diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index b870bfb2d..a67d5fb20 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -93,6 +93,13 @@ public void set(List bps, String snapshotName) throws Except } } + /* + * A list of data files within a meta backup file. The meta backup file can be + * daily snapshot (meta.json) or incrementals (meta_keyspace_cf_date.json) + * + * @param meta data file to derive the list of data files. The meta data file can be meta.json or meta_keyspace_cf_date.json + * @return a list of data files (*.db) + */ public List get(final AbstractBackupPath meta) { List files = Lists.newArrayList(); @@ -124,6 +131,35 @@ public Void retriableCall() throws Exception return files; } + /* + * Determines the existence of the backup meta file. This meta file could be snapshot (meta.json) or + * incrementals (meta_keyspace_cf..json). + * + * @param backup meta file to search + * @return true if backup meta file exist, false otherwise. + */ + public Boolean doesExist(final AbstractBackupPath meta) { + try { + new RetryableCallable() { + @Override + public Void retriableCall() throws Exception { + fs.download(meta, new FileOutputStream(meta.newRestoreFile())); //download actual file to disk + return null; + } + }.call(); + + } catch (Exception e) { + logger.error("Error downloading the Meta data try with a diffrent date...", e); + } + + if (meta.newRestoreFile().exists()) { + return true; + } else { + return false; + } + + } + private void upload(final AbstractBackupPath bp) throws Exception { new RetryableCallable() diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index e22e3e6df..b884605aa 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -142,6 +142,13 @@ public Response backupIncrementals() throws Exception @GET @Path("/list") + /* + * Fetch the list of files for the requested date range. + * + * @param date range + * @param filter. The type of data files fetched. E.g. META will only fetch the dailsy snapshot meta data file (meta.json). + * @return the list of files in json format as part of the Http response body. + */ public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryParam(REST_HEADER_FILTER) @DefaultValue("") String filter) throws Exception { Date startTime; @@ -355,6 +362,16 @@ private void setRestoreKeyspaces(String keyspaces) } } + /* + * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. + * For filter of META, ONLY the daily snapshot meta file (meta.json) are accounted for, not the incremental meta file. + * In addition, we do ONLY list the name of the meta data file, not the list of data files within it. + * + * @param handle to the json response + * @param a list of all files (data (*.db), and meta data file (*.json)) from S3 for requested dates. + * @param backup meta data file filter. Currently, the only supported filter is META, all others will be ignore. + * @return a list of files in Json format. + */ private JSONObject constructJsonResponse(JSONObject object, Iterator it,String filter) throws Exception { int fileCnt = 0; @@ -377,17 +394,19 @@ private JSONObject constructJsonResponse(JSONObject object, Iterator allFiles = metaData.get(p); - long totalSize = 0; - for (AbstractBackupPath abp : allFiles) - totalSize = totalSize + abp.getSize(); - backupJSON.put("num_files", Long.toString(allFiles.size())); - // keyValues.put("TOTAL-SIZE", Long.toString(totalSize)); // - // Add Later + if ("meta".equalsIgnoreCase(filter)) { //only check for existence of meta file + p.setFileName("meta.json"); //ignore incremental meta files, we are only interested in daily snapshot + if (metaData.doesExist(p)) { + //if here, snapshot completed. + fileCnt++; + jArray.put(backupJSON); + backupJSON.put("num_files", "1"); + } + } else { //account for every file (data, and meta) . + fileCnt++; + jArray.put(backupJSON); } - fileCnt++; - jArray.put(backupJSON); + } object.put("files", jArray); object.put("num_files", fileCnt);