Merge pull request #153 from asmorodskyi/updaterun_fixing

Extend logging and fix missing namespace in where clause

ocw/lib/db.py

@@ -34,18 +34,20 @@ def sync_csp_to_local_db(pc_instances, provider, namespace):
         if i.vault_namespace != namespace:
             raise ValueError('Instance {} does not belong to {}'.format(i, namespace))
 
-        logger.debug("Update/Create instance %s:%s @ %s", provider, i.instance_id, i.region)
-        if Instance.objects.filter(provider=i.provider, instance_id=i.instance_id).exists():
-            o = Instance.objects.get(provider=i.provider, instance_id=i.instance_id)
+        if Instance.objects.filter(provider=i.provider, instance_id=i.instance_id, vault_namespace=namespace).exists():
+            logger.debug("[%s] Update instance %s:%s", namespace, provider, i.instance_id)
+            o = Instance.objects.get(provider=i.provider, instance_id=i.instance_id, vault_namespace=namespace)
             if o.region != i.region:
-                logger.info("Instance %s:%s changed region from %s to %s",
-                            provider, i.instance_id, o.region, i.region)
+                logger.info("[%s] Instance %s:%s changed region from %s to %s",
+                            namespace, provider, i.instance_id, o.region, i.region)
                 o.region = i.region
             if o.state == StateChoice.DELETED:
-                logger.error("Update already DELETED instance %s:%s\n\t%s", provider, i.instance_id, i.csp_info)
+                logger.error("[%s] %s:%s instance which still exists has DELETED state in DB. Reactivating %s",
+                             namespace, provider, i.instance_id, i.all_time_fields())
             if o.state != StateChoice.DELETING:
                 o.state = StateChoice.ACTIVE
         else:
+            logger.debug("[%s] Create instance %s:%s", namespace, provider, i.instance_id)
             o = Instance(
                 provider=provider,
                 vault_namespace=namespace,
@@ -205,7 +207,7 @@ def update_run():
                 try:
                     _update_provider(provider, namespace)
                 except Exception:
-                    logger.exception("[{}] Update failed for {}".format(namespace, provider))
+                    logger.exception("[%s] Update failed for %s", namespace, provider)
                     email_text.add(traceback.format_exc())
                     time.sleep(5)
                 else:
@@ -226,8 +228,7 @@ def update_run():
 
 
 def delete_instance(instance):
-    logger.debug("[{}][{}] Delete instance {}".format(
-        instance.provider, instance.vault_namespace, instance.instance_id))
+    logger.debug("[%s] Delete instance %s:%s", instance.vault_namespace, instance.provider, instance.instance_id)
     if (instance.provider == ProviderChoice.AZURE):
         Azure(instance.vault_namespace).delete_resource(instance.instance_id)
     elif (instance.provider == ProviderChoice.EC2):
@@ -249,11 +250,12 @@ def auto_delete_instances():
                      age__gte=F('ttl')).exclude(csp_info__icontains='pcw_ignore')
         email_text = set()
         for i in o:
-            logger.info("[{}][{}] TTL expire for instance {}".format(i.provider, i.vault_namespace, i.instance_id))
+            logger.info("[%s] TTL expire for instance %s:%s %s", i.vault_namespace,
+                        i.provider, i.instance_id, i.all_time_fields())
             try:
                 delete_instance(i)
             except Exception:
-                msg = "[{}][{}] Deleting instance ({}) failed".format(i.provider, i.vault_namespace, i.instance_id)
+                msg = "[{}] Deleting instance ({}:{}) failed".format(i.vault_namespace, i.provider, i.instance_id)
                 logger.exception(msg)
                 email_text.add("{}\n\n{}".format(msg, traceback.format_exc()))
 

ocw/migrations/0006_auto_20220720_1711.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1.4 on 2022-07-20 17:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ocw', '0005_instance_max_age'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='instance',
+            unique_together={('provider', 'instance_id', 'vault_namespace')},
+        ),
+    ]

ocw/migrations/0007_auto_20220720_1742.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2022-07-20 17:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ocw', '0006_auto_20220720_1711'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='instance_id',
+            field=models.CharField(max_length=200),
+        ),
+    ]

ocw/models.py

@@ -53,7 +53,7 @@ class Instance(models.Model):
     active = models.BooleanField(default=False, help_text='True if the last sync found this instance on CSP')
     state = models.CharField(max_length=8, default=StateChoice.UNK, choices=StateChoice.choices(),
                              help_text='Local computed state of that Instance')
-    instance_id = models.CharField(max_length=200, unique=True)
+    instance_id = models.CharField(max_length=200)
     region = models.CharField(max_length=64, default='')
     vault_namespace = models.CharField(max_length=64, default='')
     csp_info = models.TextField(default='')
@@ -65,6 +65,12 @@ def age_formated(self):
     def ttl_formated(self):
         return format_seconds(self.ttl.total_seconds()) if(self.ttl) else ""
 
+    def all_time_fields(self):
+        all_time_pattern = "(age={}, first_seen={}, last_seen={}, ttl={})"
+        first_fmt = self.first_seen.strftime('%Y-%m-%d %H:%M')
+        last_fmt = self.last_seen.strftime('%Y-%m-%d %H:%M')
+        return all_time_pattern.format(self.age_formated(), first_fmt, last_fmt, self.ttl_formated())
+
     def tags(self):
         try:
             info = json.loads(self.csp_info)
@@ -83,4 +89,4 @@ def get_openqa_job_link(self):
         return None
 
     class Meta:
-        unique_together = (('provider', 'instance_id'),)
+        unique_together = (('provider', 'instance_id', 'vault_namespace'),)