trying to fix pipelining

also trying to understand it.

according to python:
1) you send a request
2) you MUST get response headers for (1) (THIS IS MANDATORY)
3) you send another request
4) you get response body for (2)
5) response headers for (3)
6) response body for (5)

Only two requests can be pipelined. Surely this is an unavoidable, wait no it's just written into the code to error out if you don't do it that way.

according to reality:
1) you send a request
2) you do not get response headers for (1)
3) you repeat steps 1-2 until enough responses are queued
4) you receive those responses as header,body,header,body...

they even name it with a __ so to make it hard to override, but the state can safely go to Idle after a request has sent, whether or not response headers have come in. Sure the connection might close, but then you adjust to not pipeline, and re-send the rest of your requests over a new connection.

update_from_db.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-from http.client import HTTPConnection,HTTPException,BadStatusLine
+from http.client import HTTPConnection,HTTPException,BadStatusLine,_CS_IDLE
 import json
 import base64
 from contextlib import closing
@@ -35,57 +35,72 @@ def maybeReplace(location,base,encoding=None):
         return replace(location,base,encoding=encoding,path=path)(handle)
     return deco
 
+class Penguin:
+    "idk"
+    def __init__(self, url, recv, diemessage):
+        self.url = url
+        self.recv = recv
+        self.diemessage = diemessage		
+
 class Pipeline(list):
     "Gawd why am I being so elaborate?"
     def __init__(self, threshold=10):
         "threshold is how many requests in parallel to pipeline"
         self.threshold = threshold
         self.sent = True
-    def __enter__(self,*a):
+    def __enter__(self):
         self.reopen()
-    def __exit__(self):
+        return self
+    def __exit__(self,typ,exn,trace):
+        self.send()
         self.drain()
     def reopen(self):
         self.c = HTTPConnection(server)
         self.send()
     def append(self,url,recv,diemessage):
-        super().append((url,recv,diemessage))
+        self.sent = False
+        super().append(Penguin(url,recv,diemessage))
         if len(self) > self.threshold:			
             self.send()
             self.drain()
     def trydrain(self):		
-        for url,recv,diemessage in self:
+        for penguin in self:
             try:
-                recv(self.c)
+                penguin.response.begin()
+                penguin.recv(penguin.response)
             except BadStatusLine as e:
+                print('derped requesting',penguin.url)
                 return False			
             except HTTPException as e:
-                die(diemessage+' (url='+url+')')
+                die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')')
             self.clear()
             return True
     def drain(self):
         print('draining pipeline...')
         assert self.sent, "Can't drain without sending the requests!"
         self.sent = False
-        while trydrain() is not True:
+        while self.trydrain() is not True:
             self.c.close()
-            print('derped requesting',url)
             print('drain failed, trying again')
             time.sleep(1)
             self.reopen()
     def trysend(self):
-        for url,_,diemessage in pipeline:
+        for penguin in pipeline:
             try:
-                self.c.request("GET", url)
+                self.c.request("GET", penguin.url)
+                self.c._HTTPConnection__state = _CS_IDLE
+                penguin.response = self.c.response_class(self.c.sock,
+                                                         method="GET")
+                # begin LATER so we can send multiple requests w/out response headers
             except BadStatusLine:
                 return False
             except HTTPException as e:
-                die(diemessage)
+                die(diemessage+' because of a '+repr(e))
         return True
     def send(self):
         if self.sent: return
         print('filling pipeline...')
-        while self.tryresend() is not True:
+        while self.trysend() is not True:
             self.c.close()
             print('derped resending')
             time.sleep(1)
@@ -137,23 +152,21 @@ with Pipeline() as pipeline:
                 f.write(str(s["author"]) + '\n')
                 f.write(str(s["license"]))
             url = "/skins/1/" + str(s["id"]) + ".png"
-            def tryget(c):			
-                with closing(c.getresponse()) as r:
-                    if r.status != 200:
-                        print("Error", r.status)
-                        return
-                    @replace(skinsdir,previewbase,path=preview)
-                    def go(f):
-                        shutil.copyfileobj(r,f)
-                
+            def tryget(r):			
+                if r.status != 200:
+                    print("Error", r.status)
+                    return
+                @replace(skinsdir,previewbase,path=preview)
+                def go(f):
+                    shutil.copyfileobj(r,f)
+                    
             pipeline.append(url,tryget,
-                            "Couldn't get {} because of a {}".format(
-                                s["id"],
-                                e))
+                            "Couldn't get {} because of a".format(
+                                s["id"]))
         if not foundOne:
             print("No skins updated on this page. Seems we're done?")
             #raise SystemExit
-    addpage(1)
+    addpage(curpage)
     while pages > curpage:
         curpage = curpage + 1
         addpage(curpage)