#44 - Load data correctly from servers that send all the data in one TCP packet.

Known examples include tildeverse.org and gemini.conman.org.  The technique is simply reading the whole response into a buffer, rather than reading one packet and assuming it's the header.
1 files changed, 33 insertions(+), 42 deletions(-)

M src/main/java/com/ajtjp/geminiclient/GeminiClient.java
M src/main/java/com/ajtjp/geminiclient/GeminiClient.java +33 -42
@@ 45,7 45,12 @@ public class GeminiClient {
         os.flush();
         
         InputStream is = s.getInputStream();
-        doc.setHeaders(readHeaders(is));
+        
+        byte[] serverResponse = fetchServerResponse(is);
+        
+        String headers = new String(serverResponse, 0, Math.min(serverResponse.length, 1024), "UTF-8");
+        int endOfHeaders = headers.indexOf("\r\n");
+        doc.setHeaders(headers.substring(0, endOfHeaders));
         
         if (doc.statusCode == 30 || doc.statusCode == 31) {
             //TODO: Redirect loop detection

          
@@ 54,47 59,43 @@ public class GeminiClient {
             return connect(doc.meta, 1965);
         }
         
+        int bodyLength = serverResponse.length - (endOfHeaders + 2);
+        
         if (doc.getMeta().startsWith("text/")) {
             //TODO: Should this apply for text/xml; text/plain; etc. as well
             //as text/gemini?  Quite possible not.  Leaving it for now as I
             //haven't seen any issues with pretending text/plain; is
             //text/gemini;, yet, and non-text; needs more support first.
-            StringBuilder sb = new StringBuilder();
-            byte[] buffer = new byte[2000];
-
-            while (true) {
-                int bytesRead = is.read(buffer, 0, 2000);
-                if (bytesRead == -1) {
-                    doc.lines = parseLines(sb.toString());
-                    return doc;
-                }
-                sb.append(new String(buffer, 0, bytesRead, "UTF-8"));
-            }
+            doc.lines = parseLines(new String(serverResponse, endOfHeaders + 2, bodyLength, "UTF-8"));
         }
         else {
-            byte[] fileContents = new byte[4096];
-            byte[] buffer = new byte[2000];
-            int fileContentsIndex = 0;
+            byte[] bodyContents = new byte[bodyLength];
+            System.arraycopy(serverResponse, endOfHeaders + 2, bodyContents, 0, bodyLength);
+            doc.setBinaryContents(bodyContents);
+        }
+        return doc;        
+    }
 
-            while (true) {
-                int bytesRead = is.read(buffer, 0, 2000);
-                if (bytesRead == -1) {
-                    fileContents = trimBufferToSize(fileContents, fileContentsIndex);
-                    
-                    doc.setBinaryContents(fileContents);
-                    return doc;
-                }
-                
-                //expand and re-allocate wholeBuffer if need be
-                if (fileContentsIndex + bytesRead > fileContents.length) {
-                    fileContents = expandFileContents(fileContents, fileContentsIndex);
-                }
-                
-                System.arraycopy(buffer, 0, fileContents, fileContentsIndex, bytesRead);
-                fileContentsIndex += bytesRead;
+    private static byte[] fetchServerResponse(InputStream is) throws IOException {
+        byte[] serverResponse = new byte[4096];
+        byte[] buffer = new byte[2000];
+        int fileContentsIndex = 0;
+        while (true) {
+            int bytesRead = is.read(buffer, 0, 2000);
+            if (bytesRead == -1) {
+                serverResponse = trimBufferToSize(serverResponse, fileContentsIndex);
+                break;
             }
+            
+            //expand and re-allocate wholeBuffer if need be
+            if (fileContentsIndex + bytesRead > serverResponse.length) {
+                serverResponse = expandFileContents(serverResponse, fileContentsIndex);
+            }
+            
+            System.arraycopy(buffer, 0, serverResponse, fileContentsIndex, bytesRead);
+            fileContentsIndex += bytesRead;
         }
-        
+        return serverResponse;
     }
 
     private static byte[] trimBufferToSize(byte[] fileContents, int wholeBufferIndex) {

          
@@ 128,16 129,6 @@ public class GeminiClient {
         return host;
     }
     
-    private static String readHeaders(InputStream is) throws IOException {
-        //Max size is 1029 bytes, 1024 for meta + 2 for status + 3 for whitespace
-        byte[] buffer = new byte[1200];
-        int bytesRead = is.read(buffer, 0, 1200);
-        if (bytesRead == -1) {
-            throw new IOException("Server did not send any response or response headers");
-        }
-        return new String(buffer, 0, bytesRead, "UTF-8");
-    }
-    
     private static List<GeminiLine> parseLines(String responseBody) {
         String[] rawLines = responseBody.split("\n");
         List<GeminiLine> geminiLines = new ArrayList<GeminiLine>();