Merge pull request #277 from FIRST-Tech-Challenge/pr_improved_admin_a…

…ctions

Fixed admin action for deleting blobs

server/app_engine/action.py

@@ -60,14 +60,14 @@
 ACTION_NAME_RESET_REMAINING_TRAINING_MINUTES = 'reset_remaining_training_minutes'
 ACTION_NAME_INCREMENT_REMAINING_TRAINING_MINUTES = 'increment_remaining_training_minutes'
 ACTION_NAME_SAVE_END_OF_SEASON_ENTITIES = 'save_end_of_season_entities'
-ACTION_NAME_EXPUNGE_STORAGE = 'expunge_storage'
+ACTION_NAME_RESET_TEAM_ENTITIES = 'reset_team_entities'
 ACTION_NAME_EXPUNGE_BLOB_STORAGE = 'expunge_blob_storage'
 
 def create_action_parameters(team_uuid, action_name):
     if (action_name == ACTION_NAME_RESET_REMAINING_TRAINING_MINUTES or
             action_name == ACTION_NAME_INCREMENT_REMAINING_TRAINING_MINUTES or
             action_name == ACTION_NAME_SAVE_END_OF_SEASON_ENTITIES or
-            action_name == ACTION_NAME_EXPUNGE_STORAGE or
+            action_name == ACTION_NAME_RESET_TEAM_ENTITIES or
             action_name == ACTION_NAME_EXPUNGE_BLOB_STORAGE or
             action_name == ACTION_NAME_TEST):
         is_admin_action = True

server/app_engine/app_engine.py

@@ -173,11 +173,34 @@ def validate_string_not_empty(s):
     raise exceptions.HttpErrorBadRequest(message)
 
 
+def validate_team_uuid_prefixes(s):
+    team_uuid_prefixes = []
+    tokens = s.split(',')
+    valid = True
+    allowed = '0123456789abcdef'
+    for token in tokens:
+      if len(token) > 32:
+          valid = False
+          break
+      for c in token:
+          if c not in allowed:
+              valid = False
+              break
+      if not valid:
+          break
+      team_uuid_prefixes.append(token)
+    if valid:
+        return team_uuid_prefixes
+    message = "Error: '%s' is not a valid argument." % s
+    logging.critical(message)
+    raise exceptions.HttpErrorBadRequest(message)
+
+
 def validate_boolean(s):
     if s == 'false':
-        return False;
+        return False
     if s == 'true':
-        return True;
+        return True
     message = "Error: '%s' is not a valid boolean argument." % s
     logging.critical(message)
     raise exceptions.HttpErrorBadRequest(message)
@@ -1595,36 +1618,52 @@ def save_end_of_season_entities():
     return flask.jsonify(__sanitize(response))
 
 
-@app.route('/expungeData', methods=['POST'])
+@app.route('/resetTeamEntities', methods=['POST'])
 @handle_exceptions
 @login_required
 @roles_accepted(roles.Role.GLOBAL_ADMIN, roles.Role.ML_DEVELOPER)
-def expunge_data():
+def resetTeamEntities():
     data = validate_keys(flask.request.form.to_dict(flat=True),
-        ['keep_team_entities', 'keep_tflite_and_labels', 'date_time_string'])
-    keep_team_entities = validate_boolean(data.get('keep_team_entities'))
-    keep_tflite_and_labels = validate_boolean(data.get('keep_tflite_and_labels'))
-
+        ['date_time_string'])
+    # Deleting Video, VideoFrame, Tracker, TrackerClient, Dataset, DatasetRecordWriter,
+    # DatasetRecord, DatasetZipper, Model, ModelSummaryItems, and Action collections should be done
+    # in the cloud console which is much faster and easier than doing it in python.
     action_parameters = action.create_action_parameters(
-        '', action.ACTION_NAME_EXPUNGE_STORAGE)
+        '', action.ACTION_NAME_RESET_TEAM_ENTITIES)
     action_parameters['date_time_string'] = data.get('date_time_string')
-    action_parameters['keep_team_entities'] = keep_team_entities
     action_parameters['num_teams_updated'] = 0
     action_parameters['teams_updated'] = []
     action_parameters['failure_counts'] = {}
-    action_parameters['num_entities_deleted'] = 0
-    storage_action_uuid = action.trigger_action_via_blob(action_parameters)
+    action_uuid = action.trigger_action_via_blob(action_parameters)
+    response = {
+        'action_uuid': action_uuid,
+    }
+    return flask.jsonify(__sanitize(response))
+
+@app.route('/expungeBlobStorage', methods=['POST'])
+@handle_exceptions
+@login_required
+@roles_accepted(roles.Role.GLOBAL_ADMIN, roles.Role.ML_DEVELOPER)
+def expunge_blob_storage():
+    data = validate_keys(flask.request.form.to_dict(flat=True),
+        ['date_time_string', 'keep_tflite_and_labels', 'team_uuid_prefixes'])
+    date_time_string = data.get('date_time_string')
+    keep_tflite_and_labels = validate_boolean(data.get('keep_tflite_and_labels'))
+    team_uuid_prefixes = validate_team_uuid_prefixes(data.get('team_uuid_prefixes'))
+    action_uuids = []
+    for team_uuid_prefix in team_uuid_prefixes:
+        action_parameters = action.create_action_parameters(
+            '', action.ACTION_NAME_EXPUNGE_BLOB_STORAGE)
+        action_parameters['date_time_string'] = date_time_string
+        action_parameters['keep_tflite_and_labels'] = keep_tflite_and_labels
+        action_parameters['team_uuid_prefix'] = team_uuid_prefix
+        action_parameters['num_blobs_deleted'] = 0
+        action_parameters['num_blobs_not_deleted'] = 0
+        action_uuid = action.trigger_action_via_blob(action_parameters)
+        action_uuids.append(action_uuid)
 
-    action_parameters = action.create_action_parameters(
-        '', action.ACTION_NAME_EXPUNGE_BLOB_STORAGE)
-    action_parameters['date_time_string'] = data.get('date_time_string')
-    action_parameters['keep_tflite_and_labels'] = keep_tflite_and_labels
-    action_parameters['num_blobs_deleted'] = 0
-    action_parameters['num_blobs_not_deleted'] = 0
-    blob_storage_action_uuid = action.trigger_action_via_blob(action_parameters)
     response = {
-        'storage_action_uuid': storage_action_uuid,
-        'blob_storage_action_uuid': blob_storage_action_uuid,
+        'action_uuids': action_uuids,
     }
     return flask.jsonify(__sanitize(response))
 

server/app_engine/blob_storage.py

@@ -375,50 +375,50 @@ def delete_model_blobs(model_folder, action_parameters=None):
 
 def expunge_blob_storage(action_parameters):
     keep_tflite_and_labels = action_parameters['keep_tflite_and_labels']
+    blob_name_prefix = action_parameters['team_uuid_prefix']
     client = util.storage_client()
     if 'max_results' not in action_parameters:
         action_parameters['max_results'] = 500
     max_results = action_parameters['max_results']
     while True:
         action.retrigger_if_necessary(action_parameters)
-        logging.info('expunge_blob_storage - max_results is %d' % max_results)
+        logging.info('expunge_blob_storage for %s - max_results is %d' % (blob_name_prefix, max_results))
         action.retrigger_if_necessary(action_parameters)
         count_blobs = 0
         count_blobs_to_ignore = 0
         blob_names_to_delete = []
-        for blob in client.list_blobs(BUCKET_BLOBS, max_results=max_results):
+        for blob in client.list_blobs(BUCKET_BLOBS, prefix=blob_name_prefix, max_results=max_results):
             action.retrigger_if_necessary(action_parameters)
             count_blobs += 1
             # Don't delete blobs whose names begin with team_info/
             if blob.name.startswith("team_info/"):
                 count_blobs_to_ignore += 1
-                continue;
+                continue
             if keep_tflite_and_labels:
                 # Don't delete blobs whose names end in /tflite/model_with_metadata.tflite
                 if blob.name.endswith('/tflite/model_with_metadata.tflite'):
                     count_blobs_to_ignore += 1
-                    continue;
+                    continue
                 # Don't delete blobs whose names end in /tflite/label_map.txt
                 if blob.name.endswith('/tflite/label_map.txt'):
                     count_blobs_to_ignore += 1
-                    continue;
+                    continue
             blob_names_to_delete.append(blob.name)
         action.retrigger_if_necessary(action_parameters)
-        logging.info('expunge_blob_storage - found %d blobs' % count_blobs)
-        logging.info('expunge_blob_storage - ignoring %d blobs' % count_blobs_to_ignore)
+        logging.info('expunge_blob_storage for %s - found %d blobs' % (blob_name_prefix, count_blobs))
+        logging.info('expunge_blob_storage for %s - ignoring %d blobs' % (blob_name_prefix, count_blobs_to_ignore))
         if len(blob_names_to_delete) > 0:
             # We found some blobs to delete.
-            logging.info('expunge_blob_storage - deleting %d blobs' % len(blob_names_to_delete))
+            logging.info('expunge_blob_storage for %s - deleting %d blobs' % (blob_name_prefix, len(blob_names_to_delete)))
             __delete_blobs(blob_names_to_delete)
             action_parameters['num_blobs_deleted'] += len(blob_names_to_delete)
         elif count_blobs < max_results:
             # We didn't find any blobs to delete and we looked at all the blobs.
             action_parameters['num_blobs_not_deleted'] = count_blobs_to_ignore
             break
-        else:
-            # We didn't look at all the blobs because of the max_results, increase it to make sure
-            # we look at all the blobs.
-            max_results += count_blobs_to_ignore + 500
+        if count_blobs_to_ignore > 0:
+            # Set max_results so we look at 500 more blobs than we ignore.
+            max_results = count_blobs_to_ignore + 500
             action_parameters['max_results'] = max_results
-            logging.info('expunge_blob_storage - incrementing max_results to %d' % max_results)
-    logging.info('expunge_blob_storage - all done!')
+            logging.info('expunge_blob_storage for %s - incrementing max_results to %d' % (blob_name_prefix, max_results))
+    logging.info('expunge_blob_storage for %s - all done!' % blob_name_prefix)

server/app_engine/storage.py

@@ -2049,22 +2049,8 @@ def __save_end_of_season_entity(season, team_entity):
         return num_models
 
 
-def expunge_storage(action_parameters):
-    keep_team_entities = action_parameters['keep_team_entities']
-    if keep_team_entities:
-        __reset_team_entities(action_parameters)
-    else:
-        __expunge_entities(DS_KIND_TEAM, action_parameters)
-    __expunge_entities(DS_KIND_VIDEO, action_parameters)
-    __expunge_entities(DS_KIND_VIDEO_FRAME, action_parameters)
-    __expunge_entities(DS_KIND_DATASET, action_parameters)
-    __expunge_entities(DS_KIND_DATASET_RECORD, action_parameters)
-    __expunge_entities(DS_KIND_MODEL, action_parameters)
-    __expunge_entities(DS_KIND_MODEL_SUMMARY_ITEMS, action_parameters)
-    logging.info('expunge_storage - all done!')
-
-
-def __reset_team_entities(action_parameters):
+def reset_team_entities(action_parameters):
+    logging.info('reset_team_entities')
     datastore_client = datastore.Client()
     loop = True
     while loop:
@@ -2091,7 +2077,7 @@ def __reset_team_entities(action_parameters):
                 try:
                     datastore_client.put(team_entity)
                 except:
-                    logging.critical('__reset_team_entities - exception!!! team_key: %s traceback: %s' %
+                    logging.critical('reset_team_entities - exception!!! team_key: %s traceback: %s' %
                         (team_key, traceback.format_exc().replace('\n', ' ... ')))
                     if team_key not in action_parameters['failure_counts']:
                         action_parameters['failure_counts'][team_key] = 1
@@ -2104,24 +2090,4 @@ def __reset_team_entities(action_parameters):
                 action_parameters['teams_updated'].append(team_key)
                 action_parameters['num_teams_updated'] += 1
                 action_parameters['failure_counts'].pop(team_key, None)
-
-
-def __expunge_entities(kind, action_parameters):
-    logging.info('expunge_storage - %s' % kind)
-    datastore_client = datastore.Client()
-    # Delete the entities, 500 at a time.
-    while True:
-        action.retrigger_if_necessary(action_parameters)
-        query = datastore_client.query(kind=kind)
-        entities = list(query.fetch(500))
-        if len(entities) == 0:
-            break
-        action.retrigger_if_necessary(action_parameters)
-        keys = []
-        while len(entities) > 0:
-            entity = entities.pop()
-            keys.append(entity.key)
-        if len(keys) > 0:
-            logging.info('expunge_storage - deleting %d %s entities' % (len(keys), kind))
-            datastore_client.delete_multi(keys)
-            action_parameters['num_entities_deleted'] += len(keys)
+    logging.info('reset_team_entities - all done!')

server/app_engine/templates/admin.html

@@ -87,24 +87,69 @@ <h2>Save end-of-season entities for all teams</h2>
 
     <br><hr><br>
 
-    <h2>Expunge storage and blob storage</h2>
-    <div><input type="checkbox" id="keepTeamEntitiesCheckbox" checked="true" />
-      <label for="keepTeamEntitiesCheckbox" class="text-22">Keep Team entities (reset them, but do not delete them)</label>
+    <h2>Expunge storage (Firestore)</h2>
+    <div>Please use the Cloud Console to delete the following Firestore collections:<ul>
+        <li>Action</li>
+        <li>Dataset</li>
+        <li>DatasetRecord</li>
+        <li>DatasetRecordWriter</li>
+        <li>DatasetZipper</li>
+        <li>Model</li>
+        <li>ModelSummaryItems</li>
+        <li>Tracker</li>
+        <li>TrackerClient</li>
+        <li>Video</li>
+        <li>VideoFrame</li>
+      </ul></div>
+    <br>
+    <button id="resetTeamEntitiesButton" class="btn btn-secondary">Reset Team Entities</button>
+    <div id="resetTeamEntitiesResponse"></div>
+    <div id="resetTeamEntitiesMonitorInfo" style="display: none">
+        The action to reset team entities has been triggered.<br><br>
+        You should monitor the AdminAction entity (in the datastore) with the <i>action_uuid</i>
+        value<ul>
+          <li>&quot;<span id="resetTeamEntitiesActionUuid" class="fw-bold"></span>&quot;</li>
+        </ul>
+        When the <i>state</i> field becomes &quot;finished&quot;, the action is either
+        finished, or it encountered an error. Check the Cloud Console for logs.
+    </div>
+
+    <br><hr><br>
+
+    <h2>Expunge blob storage</h2>
+    <h4>Team UUID Prefixes</h4>
+    <div>
+      <input type="checkbox" id="team_uuid_0" checked="true" />&nbsp;<label for="team_uuid_0" class="text-22">0</label><br>
+      <input type="checkbox" id="team_uuid_1" checked="true" />&nbsp;<label for="team_uuid_1" class="text-22">1</label><br>
+      <input type="checkbox" id="team_uuid_2" checked="true" />&nbsp;<label for="team_uuid_2" class="text-22">2</label><br>
+      <input type="checkbox" id="team_uuid_3" checked="true" />&nbsp;<label for="team_uuid_3" class="text-22">3</label><br>
+      <input type="checkbox" id="team_uuid_4" checked="true" />&nbsp;<label for="team_uuid_4" class="text-22">4</label><br>
+      <input type="checkbox" id="team_uuid_5" checked="true" />&nbsp;<label for="team_uuid_5" class="text-22">5</label><br>
+      <input type="checkbox" id="team_uuid_6" checked="true" />&nbsp;<label for="team_uuid_6" class="text-22">6</label><br>
+      <input type="checkbox" id="team_uuid_7" checked="true" />&nbsp;<label for="team_uuid_7" class="text-22">7</label><br>
+      <input type="checkbox" id="team_uuid_8" checked="true" />&nbsp;<label for="team_uuid_8" class="text-22">8</label><br>
+      <input type="checkbox" id="team_uuid_9" checked="true" />&nbsp;<label for="team_uuid_9" class="text-22">9</label><br>
+      <input type="checkbox" id="team_uuid_a" checked="true" />&nbsp;<label for="team_uuid_a" class="text-22">a</label><br>
+      <input type="checkbox" id="team_uuid_b" checked="true" />&nbsp;<label for="team_uuid_b" class="text-22">b</label><br>
+      <input type="checkbox" id="team_uuid_c" checked="true" />&nbsp;<label for="team_uuid_c" class="text-22">c</label><br>
+      <input type="checkbox" id="team_uuid_d" checked="true" />&nbsp;<label for="team_uuid_d" class="text-22">d</label><br>
+      <input type="checkbox" id="team_uuid_e" checked="true" />&nbsp;<label for="team_uuid_e" class="text-22">e</label><br>
+      <input type="checkbox" id="team_uuid_f" checked="true" />&nbsp;<label for="team_uuid_f" class="text-22">f</label><br>
     </div>
     <div><input type="checkbox" id="keepTfLiteFilesCheckbox" checked="true" />
       <label for="keepTfLiteFilesCheckbox" class="text-22">Keep TFLite files (do not delete them)</label>
     </div>
     <br>
-    <button id="expungeDataButton" class="btn btn-secondary">Expunge</button>
-    <div id="expungeDataResponse"></div>
-    <div id="expungeDataMonitorInfo" style="display: none">
-        The actions to expunge storage and blob storage have been triggered.<br><br>
+    <button id="expungeBlobStorageButton" class="btn btn-secondary">Expunge Blob Storage</button>
+    <div id="expungeBlobStorageResponse"></div>
+    <div id="expungeBlobStorageMonitorInfo" style="display: none">
+        The action to expunge blob storage has been triggered.<br><br>
         You should monitor the AdminAction entities (in the datastore) with the <i>action_uuid</i>
         values<ul>
-          <li>&quot;<span id="expungeStorageActionUuid" class="fw-bold"></span>&quot;</li>
-          <li>&quot;<span id="expungeBlobStorageActionUuid" class="fw-bold"></span>&quot;</li>
+          <li><span id="expungeBlobStorageActionUuids" class="fw-bold"></span></li>
         </ul>
-        When the <i>state</i> field becomes &quot;finished&quot;, the expunging is finished.
+        When the <i>state</i> fields become &quot;finished&quot;, the expunging is either
+        finished, or it encountered an error. Check the Cloud Console for logs.
     </div>
 
     <br><hr><br>
@@ -120,25 +165,25 @@ <h2>Configuration</h2>
       <br><hr><br>
   </div>
 
-  <div class="modal" id="expungeConfirmationDialog" tabindex="-1" aria-labelledby="expungeConfirmationLabel" aria-hidden="true">
+  <div class="modal" id="confirmationDialog" tabindex="-1" aria-labelledby="confirmationLabel" aria-hidden="true">
     <div class="modal-dialog">
       <div class="modal-content"><div class="orange-border">
         <div class="modal-header">
-          <h5 class="modal-title">Expunge Storage and Blob Storage?<h5>
-          <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close" id="expungeConfirmationXButton"></button>
+          <h5 class="modal-title" id="confirmationTitle"><h5>
+          <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close" id="confirmationXButton"></button>
         </div>
         <div class="modal-body">
           <center>
-            <div id="expungeConfirmationAreYouSure"></div>
+            <div id="confirmationAreYouSure"></div>
             <div class="fw-bold">This is permanent and cannot be undone.</div>
             <br>
-            <div>To confirm, please type the following into the box below: <span class="fw-bold">Expunge</span></div>
-            <input type="text" maxlength="30" id="expungeConfirmationInput" class="form-control" style="width: 100%">
+            <div>To confirm, please type the following into the box below: <span class="fw-bold">Confirm</span></div>
+            <input type="text" maxlength="30" id="confirmationInput" class="form-control" style="width: 100%">
           </center>
         </div>
         <div class="modal-footer">
-          <button type="button" class="btn btn-secondary" data-bs-dismiss="modal" id="expungeConfirmationNoButton">Cancel</button>
-          <button type="button" class="btn btn-secondary" data-bs-dismiss="modal" id="expungeConfirmationYesButton">Expunge</button>
+          <button type="button" class="btn btn-secondary" data-bs-dismiss="modal" id="confirmationNoButton">Cancel</button>
+          <button type="button" class="btn btn-secondary" data-bs-dismiss="modal" id="confirmationYesButton">Confirm</button>
         </div>
       </div></div>
     </div>