lambda_function/main.py: Clean GG_DIR if out of space.

src/remote/lambda_function/main.py

@@ -3,6 +3,7 @@
 import os
 import sys
 import time
+import errno
 import shutil
 import subprocess as sub
 from base64 import b64decode, b64encode
@@ -20,7 +21,7 @@
     os.environ['GG_CACHE_DIR'] = "/tmp/_gg/_cache"
 
 # Now we can import gg stuff...
-from ggpaths import GGPaths, GGCache
+from ggpaths import GGPaths, GGCache, make_gg_dirs
 from common import is_executable, make_executable, run_command
 
 def is_hash_for_thunk(hash):
@@ -35,23 +36,41 @@ def handler(event, context):
     os.system("rm -rf /tmp/thunk-execute.*")
 
     # Write thunks to disk
-    for thunk_item in thunks:
-        thunk_data = b64decode(thunk_item['data'])
-        if os.path.exists(GGPaths.blob_path(thunk_item['hash'])):
-            os.remove(GGPaths.blob_path(thunk_item['hash']))
-        with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout:
-            fout.write(thunk_data)
-
-    # Move executables from Lambda package to .gg directory
-    executables_dir = os.path.join(curdir, 'executables')
-    if os.path.exists(executables_dir):
-        for exe in os.listdir(executables_dir):
-            blob_path = GGPaths.blob_path(exe)
-            exe_path = os.path.join(executables_dir, exe)
-
-            if not os.path.exists(blob_path):
-                shutil.copy(exe_path, blob_path)
-                make_executable(blob_path)
+
+    tried_once = False
+
+    while True:
+        try:
+            for thunk_item in thunks:
+                thunk_data = b64decode(thunk_item['data'])
+                if os.path.exists(GGPaths.blob_path(thunk_item['hash'])):
+                    os.remove(GGPaths.blob_path(thunk_item['hash']))
+                with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout:
+                    fout.write(thunk_data)
+
+            # Move executables from Lambda package to .gg directory
+            executables_dir = os.path.join(curdir, 'executables')
+            if os.path.exists(executables_dir):
+                for exe in os.listdir(executables_dir):
+                    blob_path = GGPaths.blob_path(exe)
+                    exe_path = os.path.join(executables_dir, exe)
+
+                    if not os.path.exists(blob_path):
+                        shutil.copy(exe_path, blob_path)
+                        make_executable(blob_path)
+
+            break
+
+        except OSError as ex:
+            if not tried_once and ex.errno == errno.ENOSPC:
+                # there's no space left; let's get rid of GG_DIR and try again
+                tried_once = True
+                os.system("rm -rf '{}'".format(GGPaths.blobs))
+                os.system("rm -rf '{}'".format(GGPaths.reductions))
+                make_gg_dirs()
+                continue
+            else:
+                raise
 
     # Execute the thunk, and upload the result
     command = ["gg-execute-static",