diff --git a/collect_app/build.gradle b/collect_app/build.gradle index c2d60eeaf80..96167a6e177 100644 --- a/collect_app/build.gradle +++ b/collect_app/build.gradle @@ -205,6 +205,7 @@ dependencies { exclude group: 'org.apache.httpcomponents' } + implementation group: 'com.evernote', name: 'android-job', version: '1.2.5' implementation "com.rarepebble:colorpicker:2.3.1" implementation "commons-io:commons-io:2.6" implementation "net.sf.kxml:kxml2:2.3.0" diff --git a/collect_app/src/main/java/org/odk/collect/android/activities/FormDownloadList.java b/collect_app/src/main/java/org/odk/collect/android/activities/FormDownloadList.java index b24a529fae2..5c569edc5d6 100644 --- a/collect_app/src/main/java/org/odk/collect/android/activities/FormDownloadList.java +++ b/collect_app/src/main/java/org/odk/collect/android/activities/FormDownloadList.java @@ -35,7 +35,7 @@ import org.odk.collect.android.adapters.FormDownloadListAdapter; import org.odk.collect.android.application.Collect; import org.odk.collect.android.dao.FormsDao; -import org.odk.collect.android.listeners.FormDownloaderListener; +import org.odk.collect.android.listeners.DownloadFormsTaskListener; import org.odk.collect.android.listeners.FormListDownloaderListener; import org.odk.collect.android.logic.FormDetails; import org.odk.collect.android.tasks.DownloadFormListTask; @@ -53,6 +53,9 @@ import timber.log.Timber; +import static org.odk.collect.android.utilities.DownloadFormListUtils.DL_AUTH_REQUIRED; +import static org.odk.collect.android.utilities.DownloadFormListUtils.DL_ERROR_MSG; + /** * Responsible for displaying, adding and deleting all the valid forms in the forms directory. One * caveat. If the server requires authentication, a dialog will pop up asking when you request the @@ -69,13 +72,14 @@ * @author Carl Hartung (carlhartung@gmail.com) */ public class FormDownloadList extends FormListActivity implements FormListDownloaderListener, - FormDownloaderListener, AuthDialogUtility.AuthDialogUtilityResultListener, AdapterView.OnItemClickListener { + DownloadFormsTaskListener, AuthDialogUtility.AuthDialogUtilityResultListener, AdapterView.OnItemClickListener { private static final String FORM_DOWNLOAD_LIST_SORTING_ORDER = "formDownloadListSortingOrder"; private static final int PROGRESS_DIALOG = 1; private static final int AUTH_DIALOG = 2; private static final int CANCELLATION_DIALOG = 3; + public static final String DISPLAY_ONLY_UPDATED_FORMS = "displayOnlyUpdatedForms"; private static final String BUNDLE_SELECTED_COUNT = "selectedcount"; private static final String BUNDLE_FORM_MAP = "formmap"; private static final String DIALOG_TITLE = "dialogtitle"; @@ -114,6 +118,7 @@ public class FormDownloadList extends FormListActivity implements FormListDownlo private boolean shouldExit; private static final String SHOULD_EXIT = "shouldexit"; + private boolean displayOnlyUpdatedForms; @SuppressWarnings("unchecked") @Override @@ -122,6 +127,10 @@ public void onCreate(Bundle savedInstanceState) { setContentView(R.layout.remote_file_manage_list); setTitle(getString(R.string.get_forms)); + if (getIntent().getExtras() != null) { + displayOnlyUpdatedForms = (boolean) getIntent().getExtras().get(DISPLAY_ONLY_UPDATED_FORMS); + } + alertMsg = getString(R.string.please_wait); downloadButton = findViewById(R.id.add_button); @@ -579,14 +588,14 @@ public void formListDownloadingComplete(HashMap result) { return; } - if (result.containsKey(DownloadFormListTask.DL_AUTH_REQUIRED)) { + if (result.containsKey(DL_AUTH_REQUIRED)) { // need authorization showDialog(AUTH_DIALOG); - } else if (result.containsKey(DownloadFormListTask.DL_ERROR_MSG)) { + } else if (result.containsKey(DL_ERROR_MSG)) { // Download failed String dialogMessage = getString(R.string.list_failed_with_error, - result.get(DownloadFormListTask.DL_ERROR_MSG).getErrorStr()); + result.get(DL_ERROR_MSG).getErrorStr()); String dialogTitle = getString(R.string.load_remote_form_error); createAlertDialog(dialogTitle, dialogMessage, DO_NOT_EXIT); } else { @@ -599,28 +608,31 @@ public void formListDownloadingComplete(HashMap result) { for (int i = 0; i < result.size(); i++) { String formDetailsKey = ids.get(i); FormDetails details = formNamesAndURLs.get(formDetailsKey); - HashMap item = new HashMap(); - item.put(FORMNAME, details.getFormName()); - item.put(FORMID_DISPLAY, - ((details.getFormVersion() == null) ? "" : (getString(R.string.version) + " " - + details.getFormVersion() + " ")) + "ID: " + details.getFormID()); - item.put(FORMDETAIL_KEY, formDetailsKey); - item.put(FORM_ID_KEY, details.getFormID()); - item.put(FORM_VERSION_KEY, details.getFormVersion()); - - // Insert the new form in alphabetical order. - if (formList.size() == 0) { - formList.add(item); - } else { - int j; - for (j = 0; j < formList.size(); j++) { - HashMap compareMe = formList.get(j); - String name = compareMe.get(FORMNAME); - if (name.compareTo(formNamesAndURLs.get(ids.get(i)).getFormName()) > 0) { - break; + + if (!displayOnlyUpdatedForms || (details.isNewerFormVersionAvailable() || details.areNewerMediaFilesAvailable())) { + HashMap item = new HashMap(); + item.put(FORMNAME, details.getFormName()); + item.put(FORMID_DISPLAY, + ((details.getFormVersion() == null) ? "" : (getString(R.string.version) + " " + + details.getFormVersion() + " ")) + "ID: " + details.getFormID()); + item.put(FORMDETAIL_KEY, formDetailsKey); + item.put(FORM_ID_KEY, details.getFormID()); + item.put(FORM_VERSION_KEY, details.getFormVersion()); + + // Insert the new form in alphabetical order. + if (formList.size() == 0) { + formList.add(item); + } else { + int j; + for (j = 0; j < formList.size(); j++) { + HashMap compareMe = formList.get(j); + String name = compareMe.get(FORMNAME); + if (name.compareTo(formNamesAndURLs.get(ids.get(i)).getFormName()) > 0) { + break; + } } + formList.add(j, item); } - formList.add(j, item); } } filteredFormList.addAll(formList); @@ -688,17 +700,21 @@ public void formsDownloadingComplete(HashMap result) { progressDialog.dismiss(); } + createAlertDialog(getString(R.string.download_forms_result), getDownloadResultMessage(result), EXIT); + } + + public static String getDownloadResultMessage(HashMap result) { Set keys = result.keySet(); StringBuilder b = new StringBuilder(); for (FormDetails k : keys) { b.append(k.getFormName() + " (" + ((k.getFormVersion() != null) - ? (this.getString(R.string.version) + ": " + k.getFormVersion() + " ") + ? (Collect.getInstance().getString(R.string.version) + ": " + k.getFormVersion() + " ") : "") + "ID: " + k.getFormID() + ") - " + result.get(k)); b.append("\n\n"); } - createAlertDialog(getString(R.string.download_forms_result), b.toString().trim(), EXIT); + return b.toString().trim(); } @Override diff --git a/collect_app/src/main/java/org/odk/collect/android/activities/MainMenuActivity.java b/collect_app/src/main/java/org/odk/collect/android/activities/MainMenuActivity.java index fef214fff38..6199a27ab88 100644 --- a/collect_app/src/main/java/org/odk/collect/android/activities/MainMenuActivity.java +++ b/collect_app/src/main/java/org/odk/collect/android/activities/MainMenuActivity.java @@ -20,7 +20,6 @@ import android.content.DialogInterface; import android.content.Intent; import android.content.SharedPreferences; -import android.content.SharedPreferences.Editor; import android.database.ContentObserver; import android.database.Cursor; import android.os.Bundle; @@ -48,7 +47,9 @@ import org.odk.collect.android.dao.InstancesDao; import org.odk.collect.android.preferences.AdminKeys; import org.odk.collect.android.preferences.AdminPreferencesActivity; +import org.odk.collect.android.preferences.AdminSharedPreferences; import org.odk.collect.android.preferences.AutoSendPreferenceMigrator; +import org.odk.collect.android.preferences.GeneralSharedPreferences; import org.odk.collect.android.preferences.PreferenceKeys; import org.odk.collect.android.preferences.PreferencesActivity; import org.odk.collect.android.provider.InstanceProviderAPI.InstanceColumns; @@ -630,57 +631,26 @@ private boolean loadSharedPreferencesFromFile(File src) { ObjectInputStream input = null; try { input = new ObjectInputStream(new FileInputStream(src)); - Editor prefEdit = PreferenceManager.getDefaultSharedPreferences( - this).edit(); - prefEdit.clear(); + GeneralSharedPreferences.getInstance().clear(); + // first object is preferences Map entries = (Map) input.readObject(); AutoSendPreferenceMigrator.migrate(entries); for (Entry entry : entries.entrySet()) { - Object v = entry.getValue(); - String key = entry.getKey(); - - if (v instanceof Boolean) { - prefEdit.putBoolean(key, (Boolean) v); - } else if (v instanceof Float) { - prefEdit.putFloat(key, (Float) v); - } else if (v instanceof Integer) { - prefEdit.putInt(key, (Integer) v); - } else if (v instanceof Long) { - prefEdit.putLong(key, (Long) v); - } else if (v instanceof String) { - prefEdit.putString(key, ((String) v)); - } + GeneralSharedPreferences.getInstance().save(entry.getKey(), entry.getValue()); } - prefEdit.apply(); + AuthDialogUtility.setWebCredentialsFromPreferences(); + AdminSharedPreferences.getInstance().clear(); + // second object is admin options - Editor adminEdit = getSharedPreferences(AdminPreferencesActivity.ADMIN_PREFERENCES, - 0).edit(); - adminEdit.clear(); - // first object is preferences Map adminEntries = (Map) input.readObject(); for (Entry entry : adminEntries.entrySet()) { - Object v = entry.getValue(); - String key = entry.getKey(); - - if (v instanceof Boolean) { - adminEdit.putBoolean(key, (Boolean) v); - } else if (v instanceof Float) { - adminEdit.putFloat(key, (Float) v); - } else if (v instanceof Integer) { - adminEdit.putInt(key, (Integer) v); - } else if (v instanceof Long) { - adminEdit.putLong(key, (Long) v); - } else if (v instanceof String) { - adminEdit.putString(key, ((String) v)); - } + AdminSharedPreferences.getInstance().save(entry.getKey(), entry.getValue()); } - adminEdit.apply(); - res = true; } catch (IOException | ClassNotFoundException e) { Timber.e(e, "Exception while loading preferences from file due to : %s ", e.getMessage()); diff --git a/collect_app/src/main/java/org/odk/collect/android/application/Collect.java b/collect_app/src/main/java/org/odk/collect/android/application/Collect.java index b69413c8bcb..7855245ea46 100644 --- a/collect_app/src/main/java/org/odk/collect/android/application/Collect.java +++ b/collect_app/src/main/java/org/odk/collect/android/application/Collect.java @@ -30,6 +30,8 @@ import android.view.View; import android.view.inputmethod.InputMethodManager; +import com.evernote.android.job.JobManager; +import com.evernote.android.job.JobManagerCreateException; import com.crashlytics.android.Crashlytics; import com.google.android.gms.analytics.GoogleAnalytics; import com.google.android.gms.analytics.Tracker; @@ -52,6 +54,7 @@ import org.odk.collect.android.utilities.AuthDialogUtility; import org.odk.collect.android.utilities.LocaleHelper; import org.odk.collect.android.utilities.PRNGFixes; +import org.odk.collect.android.utilities.ServerPollingJobCreator; import org.opendatakit.httpclientandroidlib.client.CookieStore; import org.opendatakit.httpclientandroidlib.client.CredentialsProvider; import org.opendatakit.httpclientandroidlib.client.protocol.HttpClientContext; @@ -266,6 +269,14 @@ public void onCreate() { .build() .inject(this); + try { + JobManager + .create(this) + .addJobCreator(new ServerPollingJobCreator()); + } catch (JobManagerCreateException e) { + Timber.e(e); + } + reloadSharedPreferences(); PRNGFixes.apply(); diff --git a/collect_app/src/main/java/org/odk/collect/android/database/helpers/FormsDatabaseHelper.java b/collect_app/src/main/java/org/odk/collect/android/database/helpers/FormsDatabaseHelper.java index 40b9c6dccc8..7074d928a54 100644 --- a/collect_app/src/main/java/org/odk/collect/android/database/helpers/FormsDatabaseHelper.java +++ b/collect_app/src/main/java/org/odk/collect/android/database/helpers/FormsDatabaseHelper.java @@ -40,6 +40,7 @@ import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.JR_FORM_ID; import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.JR_VERSION; import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.LANGUAGE; +import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.LAST_DETECTED_FORM_VERSION_HASH; import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.MD5_HASH; import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.SUBMISSION_URI; @@ -50,7 +51,7 @@ public class FormsDatabaseHelper extends SQLiteOpenHelper { private static final String DATABASE_NAME = "forms.db"; public static final String FORMS_TABLE_NAME = "forms"; - private static final int DATABASE_VERSION = 5; + private static final int DATABASE_VERSION = 6; // These exist in database versions 2 and 3, but not in 4... private static final String TEMP_FORMS_TABLE_NAME = "forms_v4"; @@ -79,9 +80,11 @@ public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { success &= upgradeToVersion4(db, oldVersion); case 4: success &= upgradeToVersion5(db); + case 5: + success &= upgradeToVersion6(db); break; default: - Timber.i("Unknown version " + oldVersion); + Timber.i("Unknown version %s", oldVersion); } if (success) { @@ -276,6 +279,22 @@ private boolean upgradeToVersion5(SQLiteDatabase db) { return success; } + private boolean upgradeToVersion6(SQLiteDatabase db) { + boolean success = true; + try { + CustomSQLiteQueryBuilder + .begin(db) + .alter() + .table(FORMS_TABLE_NAME) + .addColumn(LAST_DETECTED_FORM_VERSION_HASH, "text") + .end(); + } catch (SQLiteException e) { + Timber.e(e); + success = false; + } + return success; + } + private void createFormsTable(SQLiteDatabase db, String tableName) { db.execSQL("CREATE TABLE " + tableName + " (" + _ID + " integer primary key, " @@ -292,7 +311,8 @@ private void createFormsTable(SQLiteDatabase db, String tableName) { + SUBMISSION_URI + " text, " + BASE64_RSA_PUBLIC_KEY + " text, " + JRCACHE_FILE_PATH + " text not null, " - + AUTO_SEND + " text," - + AUTO_DELETE + " text);"); + + AUTO_SEND + " text, " + + AUTO_DELETE + " text, " + + LAST_DETECTED_FORM_VERSION_HASH + " text);"); } } diff --git a/collect_app/src/main/java/org/odk/collect/android/listeners/DownloadFormsTaskListener.java b/collect_app/src/main/java/org/odk/collect/android/listeners/DownloadFormsTaskListener.java new file mode 100644 index 00000000000..f9c564e74f9 --- /dev/null +++ b/collect_app/src/main/java/org/odk/collect/android/listeners/DownloadFormsTaskListener.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2009 University of Washington + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.odk.collect.android.listeners; + +import org.odk.collect.android.logic.FormDetails; + +import java.util.HashMap; + + +/** + * @author Carl Hartung (carlhartung@gmail.com) + */ +public interface DownloadFormsTaskListener { + void formsDownloadingComplete(HashMap result); + + void progressUpdate(String currentFile, int progress, int total); + + void formsDownloadingCancelled(); +} diff --git a/collect_app/src/main/java/org/odk/collect/android/listeners/FormDownloaderListener.java b/collect_app/src/main/java/org/odk/collect/android/listeners/FormDownloaderListener.java index 9f7f715b3b1..30a0e8fb5a4 100644 --- a/collect_app/src/main/java/org/odk/collect/android/listeners/FormDownloaderListener.java +++ b/collect_app/src/main/java/org/odk/collect/android/listeners/FormDownloaderListener.java @@ -1,31 +1,24 @@ /* - * Copyright (C) 2009 University of Washington - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * + * Copyright 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.odk.collect.android.listeners; -import org.odk.collect.android.logic.FormDetails; - -import java.util.HashMap; - - -/** - * @author Carl Hartung (carlhartung@gmail.com) - */ public interface FormDownloaderListener { - void formsDownloadingComplete(HashMap result); - void progressUpdate(String currentFile, int progress, int total); + void progressUpdate(String currentFile, String progress, String total); - void formsDownloadingCancelled(); + boolean isTaskCanceled(); } diff --git a/collect_app/src/main/java/org/odk/collect/android/logic/FormDetails.java b/collect_app/src/main/java/org/odk/collect/android/logic/FormDetails.java index 9ee0869b6ad..9758115a925 100644 --- a/collect_app/src/main/java/org/odk/collect/android/logic/FormDetails.java +++ b/collect_app/src/main/java/org/odk/collect/android/logic/FormDetails.java @@ -25,6 +25,8 @@ public class FormDetails implements Serializable { private String manifestUrl; private String formID; private String formVersion; + private String hash; + private String manifestFileHash; private boolean isNewerFormVersionAvailable; private boolean areNewerMediaFilesAvailable; @@ -33,13 +35,15 @@ public FormDetails(String error) { } public FormDetails(String formName, String downloadUrl, String manifestUrl, String formID, - String formVersion, boolean isNewerFormVersionAvailable, - boolean areNewerMediaFilesAvailable) { + String formVersion, String hash, String manifestFileHash, + boolean isNewerFormVersionAvailable, boolean areNewerMediaFilesAvailable) { this.formName = formName; this.downloadUrl = downloadUrl; this.manifestUrl = manifestUrl; this.formID = formID; this.formVersion = formVersion; + this.hash = hash; + this.manifestFileHash = manifestFileHash; this.isNewerFormVersionAvailable = isNewerFormVersionAvailable; this.areNewerMediaFilesAvailable = areNewerMediaFilesAvailable; } @@ -68,6 +72,14 @@ public String getFormVersion() { return formVersion; } + public String getHash() { + return hash; + } + + public String getManifestFileHash() { + return manifestFileHash; + } + public boolean isNewerFormVersionAvailable() { return isNewerFormVersionAvailable; } diff --git a/collect_app/src/main/java/org/odk/collect/android/logic/ManifestFile.java b/collect_app/src/main/java/org/odk/collect/android/logic/ManifestFile.java new file mode 100644 index 00000000000..a224fc3e865 --- /dev/null +++ b/collect_app/src/main/java/org/odk/collect/android/logic/ManifestFile.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.odk.collect.android.logic; + +import java.util.List; + +public class ManifestFile { + private final String hash; + private final List mediaFiles; + + public ManifestFile(String hash, List mediaFiles) { + this.hash = hash; + this.mediaFiles = mediaFiles; + } + + public String getHash() { + return hash; + } + + public List getMediaFiles() { + return mediaFiles; + } +} diff --git a/collect_app/src/main/java/org/odk/collect/android/preferences/AdminKeys.java b/collect_app/src/main/java/org/odk/collect/android/preferences/AdminKeys.java index 22c87d68620..c57b584a737 100644 --- a/collect_app/src/main/java/org/odk/collect/android/preferences/AdminKeys.java +++ b/collect_app/src/main/java/org/odk/collect/android/preferences/AdminKeys.java @@ -39,6 +39,8 @@ public final class AdminKeys { private static final String KEY_CHANGE_FORM_METADATA = "change_form_metadata"; // client + private static final String KEY_PERIODIC_FORM_UPDATES_CHECK = "periodic_form_updates_check"; + private static final String KEY_AUTOMATIC_UPDATE = "automatic_update"; private static final String KEY_CHANGE_FONT_SIZE = "change_font_size"; private static final String KEY_DEFAULT_TO_FINALIZED = "default_to_finalized"; private static final String KEY_HIGH_RESOLUTION = "high_resolution"; @@ -71,6 +73,8 @@ public final class AdminKeys { ag(KEY_CHANGE_SERVER, PreferenceKeys.KEY_PROTOCOL), ag(KEY_CHANGE_FORM_METADATA, PreferenceKeys.KEY_FORM_METADATA), + ag(KEY_PERIODIC_FORM_UPDATES_CHECK, PreferenceKeys.KEY_PERIODIC_FORM_UPDATES_CHECK), + ag(KEY_AUTOMATIC_UPDATE, PreferenceKeys.KEY_AUTOMATIC_UPDATE), ag(KEY_CHANGE_FONT_SIZE, PreferenceKeys.KEY_FONT_SIZE), ag(KEY_APP_LANGUAGE, PreferenceKeys.KEY_APP_LANGUAGE), ag(KEY_DEFAULT_TO_FINALIZED, PreferenceKeys.KEY_COMPLETED_DEFAULT), @@ -121,6 +125,8 @@ public final class AdminKeys { ); static Collection formManagementKeys = Arrays.asList( + KEY_PERIODIC_FORM_UPDATES_CHECK, + KEY_AUTOMATIC_UPDATE, KEY_AUTOSEND, KEY_DELETE_AFTER_SEND, KEY_DEFAULT_TO_FINALIZED, diff --git a/collect_app/src/main/java/org/odk/collect/android/preferences/AdminSharedPreferences.java b/collect_app/src/main/java/org/odk/collect/android/preferences/AdminSharedPreferences.java index 1a015eb4d42..adf8964f968 100644 --- a/collect_app/src/main/java/org/odk/collect/android/preferences/AdminSharedPreferences.java +++ b/collect_app/src/main/java/org/odk/collect/android/preferences/AdminSharedPreferences.java @@ -75,7 +75,7 @@ public void save(String key, Object value) { editor.apply(); } - private void clear() { + public void clear() { sharedPreferences .edit() .clear() diff --git a/collect_app/src/main/java/org/odk/collect/android/preferences/FormManagementPreferences.java b/collect_app/src/main/java/org/odk/collect/android/preferences/FormManagementPreferences.java index f0d5c38c548..39bf788a4c6 100644 --- a/collect_app/src/main/java/org/odk/collect/android/preferences/FormManagementPreferences.java +++ b/collect_app/src/main/java/org/odk/collect/android/preferences/FormManagementPreferences.java @@ -21,12 +21,15 @@ import android.view.View; import org.odk.collect.android.R; +import org.odk.collect.android.tasks.ServerPollingJob; import static org.odk.collect.android.preferences.AdminKeys.ALLOW_OTHER_WAYS_OF_EDITING_FORM; +import static org.odk.collect.android.preferences.PreferenceKeys.KEY_AUTOMATIC_UPDATE; import static org.odk.collect.android.preferences.PreferenceKeys.KEY_AUTOSEND; import static org.odk.collect.android.preferences.PreferenceKeys.KEY_CONSTRAINT_BEHAVIOR; import static org.odk.collect.android.preferences.PreferenceKeys.KEY_GUIDANCE_HINT; import static org.odk.collect.android.preferences.PreferenceKeys.KEY_IMAGE_SIZE; +import static org.odk.collect.android.preferences.PreferenceKeys.KEY_PERIODIC_FORM_UPDATES_CHECK; public class FormManagementPreferences extends BasePreferenceFragment { @@ -35,9 +38,11 @@ public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); addPreferencesFromResource(R.xml.form_management_preferences); - initConstraintBehaviorPref(); - initAutoSendPrefs(); - initImageSizePrefs(); + initListPref(KEY_PERIODIC_FORM_UPDATES_CHECK); + initPref(KEY_AUTOMATIC_UPDATE); + initListPref(KEY_CONSTRAINT_BEHAVIOR); + initListPref(KEY_AUTOSEND); + initListPref(KEY_IMAGE_SIZE); initGuidancePrefs(); } @@ -55,64 +60,38 @@ public void onDetach() { } } - - private void initConstraintBehaviorPref() { - final ListPreference pref = (ListPreference) findPreference(KEY_CONSTRAINT_BEHAVIOR); + private void initListPref(String key) { + final ListPreference pref = (ListPreference) findPreference(key); if (pref != null) { pref.setSummary(pref.getEntry()); - pref.setOnPreferenceChangeListener( - new Preference.OnPreferenceChangeListener() { - - @Override - public boolean onPreferenceChange(Preference preference, Object newValue) { - int index = ((ListPreference) preference).findIndexOfValue( - newValue.toString()); - CharSequence entry = ((ListPreference) preference).getEntries()[index]; - preference.setSummary(entry); - return true; - } - }); - pref.setEnabled((Boolean) AdminSharedPreferences.getInstance().get(ALLOW_OTHER_WAYS_OF_EDITING_FORM)); - } - } - - private void initAutoSendPrefs() { - final ListPreference autosend = (ListPreference) findPreference(KEY_AUTOSEND); - - if (autosend == null) { - return; - } - - autosend.setSummary(autosend.getEntry()); - autosend.setOnPreferenceChangeListener(new Preference.OnPreferenceChangeListener() { - @Override - public boolean onPreferenceChange(Preference preference, Object newValue) { + pref.setOnPreferenceChangeListener((preference, newValue) -> { int index = ((ListPreference) preference).findIndexOfValue(newValue.toString()); - String entry = (String) ((ListPreference) preference).getEntries()[index]; + CharSequence entry = ((ListPreference) preference).getEntries()[index]; preference.setSummary(entry); + if (key.equals(KEY_PERIODIC_FORM_UPDATES_CHECK)) { + ServerPollingJob.schedulePeriodicJob((String) newValue); + if (newValue.equals(getString(R.string.never_value))) { + findPreference(KEY_AUTOMATIC_UPDATE).setEnabled(false); + } + getActivity().recreate(); + } return true; + }); + if (key.equals(KEY_CONSTRAINT_BEHAVIOR)) { + pref.setEnabled((Boolean) AdminSharedPreferences.getInstance().get(ALLOW_OTHER_WAYS_OF_EDITING_FORM)); } - }); + } } - private void initImageSizePrefs() { - final ListPreference imageSize = (ListPreference) findPreference(KEY_IMAGE_SIZE); + private void initPref(String key) { + final Preference pref = findPreference(key); - if (imageSize == null) { - return; - } - - imageSize.setSummary(imageSize.getEntry()); - imageSize.setOnPreferenceChangeListener(new Preference.OnPreferenceChangeListener() { - @Override - public boolean onPreferenceChange(Preference preference, Object newValue) { - int index = ((ListPreference) preference).findIndexOfValue(newValue.toString()); - String entry = (String) ((ListPreference) preference).getEntries()[index]; - preference.setSummary(entry); - return true; + if (pref != null) { + if (key.equals(KEY_AUTOMATIC_UPDATE)) { + pref.setEnabled(!GeneralSharedPreferences.getInstance().get(KEY_PERIODIC_FORM_UPDATES_CHECK).equals(getString(R.string.never_value))); } - }); + } } diff --git a/collect_app/src/main/java/org/odk/collect/android/preferences/GeneralSharedPreferences.java b/collect_app/src/main/java/org/odk/collect/android/preferences/GeneralSharedPreferences.java index 3558cd3bd5e..d58d366feb8 100644 --- a/collect_app/src/main/java/org/odk/collect/android/preferences/GeneralSharedPreferences.java +++ b/collect_app/src/main/java/org/odk/collect/android/preferences/GeneralSharedPreferences.java @@ -17,6 +17,7 @@ import android.preference.PreferenceManager; import org.odk.collect.android.application.Collect; +import org.odk.collect.android.tasks.ServerPollingJob; import java.util.Map; import java.util.Set; @@ -24,6 +25,7 @@ import timber.log.Timber; import static org.odk.collect.android.preferences.PreferenceKeys.GENERAL_KEYS; +import static org.odk.collect.android.preferences.PreferenceKeys.KEY_PERIODIC_FORM_UPDATES_CHECK; public class GeneralSharedPreferences { @@ -73,7 +75,11 @@ public void reset(String key) { public GeneralSharedPreferences save(String key, Object value) { editor = sharedPreferences.edit(); + if (value == null || value instanceof String) { + if (key.equals(KEY_PERIODIC_FORM_UPDATES_CHECK) && get(KEY_PERIODIC_FORM_UPDATES_CHECK) != value) { + ServerPollingJob.schedulePeriodicJob((String) value); + } editor.putString(key, (String) value); } else if (value instanceof Boolean) { editor.putBoolean(key, (Boolean) value); diff --git a/collect_app/src/main/java/org/odk/collect/android/preferences/PreferenceKeys.java b/collect_app/src/main/java/org/odk/collect/android/preferences/PreferenceKeys.java index 8e9293d684b..c649695dca3 100644 --- a/collect_app/src/main/java/org/odk/collect/android/preferences/PreferenceKeys.java +++ b/collect_app/src/main/java/org/odk/collect/android/preferences/PreferenceKeys.java @@ -23,6 +23,8 @@ public final class PreferenceKeys { public static final String KEY_IMAGE_SIZE = "image_size"; public static final String KEY_GUIDANCE_HINT = "guidance_hint"; public static final String KEY_INSTANCE_SYNC = "instance_sync"; + public static final String KEY_PERIODIC_FORM_UPDATES_CHECK = "periodic_form_updates_check"; + public static final String KEY_AUTOMATIC_UPDATE = "automatic_update"; // form_metadata_preferences.xml public static final String KEY_METADATA_USERNAME = "metadata_username"; @@ -87,6 +89,8 @@ private static HashMap getHashMap() { hashMap.put(KEY_HIGH_RESOLUTION, true); hashMap.put(KEY_IMAGE_SIZE, "original_image_size"); hashMap.put(KEY_INSTANCE_SYNC, true); + hashMap.put(KEY_PERIODIC_FORM_UPDATES_CHECK, "never"); + hashMap.put(KEY_AUTOMATIC_UPDATE, false); // form_metadata_preferences.xml hashMap.put(KEY_METADATA_USERNAME, ""); hashMap.put(KEY_METADATA_PHONENUMBER, ""); diff --git a/collect_app/src/main/java/org/odk/collect/android/provider/FormsProvider.java b/collect_app/src/main/java/org/odk/collect/android/provider/FormsProvider.java index 0ec3e4d9e11..a55c1cdcf90 100644 --- a/collect_app/src/main/java/org/odk/collect/android/provider/FormsProvider.java +++ b/collect_app/src/main/java/org/odk/collect/android/provider/FormsProvider.java @@ -534,5 +534,6 @@ public int update(Uri uri, ContentValues values, String where, String[] whereArg sFormsProjectionMap.put(FormsColumns.LANGUAGE, FormsColumns.LANGUAGE); sFormsProjectionMap.put(FormsColumns.AUTO_DELETE, FormsColumns.AUTO_DELETE); sFormsProjectionMap.put(FormsColumns.AUTO_SEND, FormsColumns.AUTO_SEND); + sFormsProjectionMap.put(FormsColumns.LAST_DETECTED_FORM_VERSION_HASH, FormsColumns.LAST_DETECTED_FORM_VERSION_HASH); } } diff --git a/collect_app/src/main/java/org/odk/collect/android/provider/FormsProviderAPI.java b/collect_app/src/main/java/org/odk/collect/android/provider/FormsProviderAPI.java index 54de5a3e136..0fe06766f1f 100644 --- a/collect_app/src/main/java/org/odk/collect/android/provider/FormsProviderAPI.java +++ b/collect_app/src/main/java/org/odk/collect/android/provider/FormsProviderAPI.java @@ -51,6 +51,7 @@ private FormsColumns() { public static final String SUBMISSION_URI = "submissionUri"; // can be null public static final String BASE64_RSA_PUBLIC_KEY = "base64RsaPublicKey"; // can be null public static final String AUTO_DELETE = "autoDelete"; // can be null + public static final String LAST_DETECTED_FORM_VERSION_HASH = "lastDetectedFormVersionHash"; // can be null // Column is called autoSubmit for legacy support but the attribute is auto-send public static final String AUTO_SEND = "autoSubmit"; // can be null diff --git a/collect_app/src/main/java/org/odk/collect/android/receivers/NetworkReceiver.java b/collect_app/src/main/java/org/odk/collect/android/receivers/NetworkReceiver.java index cb24bd1f95b..39644dfe942 100644 --- a/collect_app/src/main/java/org/odk/collect/android/receivers/NetworkReceiver.java +++ b/collect_app/src/main/java/org/odk/collect/android/receivers/NetworkReceiver.java @@ -18,6 +18,7 @@ import org.odk.collect.android.application.Collect; import org.odk.collect.android.dao.FormsDao; import org.odk.collect.android.dao.InstancesDao; +import org.odk.collect.android.tasks.ServerPollingJob; import org.odk.collect.android.utilities.gdrive.GoogleAccountsManager; import org.odk.collect.android.listeners.InstanceUploaderListener; import org.odk.collect.android.preferences.GeneralSharedPreferences; @@ -59,6 +60,8 @@ public void onReceive(Context context, Intent intent) { && currentNetworkInfo.getState() == NetworkInfo.State.CONNECTED) { uploadForms(context, isFormAutoSendOptionEnabled(currentNetworkInfo)); } + + ServerPollingJob.pollServerIfNeeded(); } else if (action.equals("org.odk.collect.android.FormSaved")) { ConnectivityManager connectivityManager = (ConnectivityManager) context .getSystemService(Context.CONNECTIVITY_SERVICE); @@ -187,8 +190,8 @@ public void uploadingComplete(HashMap result) { StringBuilder message = new StringBuilder(); message - .append(Collect.getInstance().getString(R.string.odk_auto_note)) - .append(" :: \n\n"); + .append(Collect.getInstance().getString(R.string.forms_sent)) + .append("\n\n"); if (result == null) { message.append(Collect.getInstance().getString(R.string.odk_auth_auth_fail)); diff --git a/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormListTask.java b/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormListTask.java index 1402c750015..f0e4596d8cf 100644 --- a/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormListTask.java +++ b/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormListTask.java @@ -14,36 +14,17 @@ package org.odk.collect.android.tasks; -import android.content.SharedPreferences; import android.os.AsyncTask; -import android.preference.PreferenceManager; -import org.javarosa.xform.parse.XFormParser; -import org.kxml2.kdom.Element; -import org.odk.collect.android.R; -import org.odk.collect.android.application.Collect; -import org.odk.collect.android.dao.FormsDao; import org.odk.collect.android.listeners.FormListDownloaderListener; import org.odk.collect.android.logic.FormDetails; -import org.odk.collect.android.logic.MediaFile; -import org.odk.collect.android.preferences.PreferenceKeys; -import org.odk.collect.android.utilities.DocumentFetchResult; -import org.odk.collect.android.utilities.FileUtils; -import org.odk.collect.android.utilities.WebUtils; -import org.opendatakit.httpclientandroidlib.client.HttpClient; -import org.opendatakit.httpclientandroidlib.protocol.HttpContext; +import org.odk.collect.android.utilities.DownloadFormListUtils; -import java.io.File; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; - -import timber.log.Timber; /** * Background task for downloading forms from urls or a formlist from a url. We overload this task - * a - * bit so that we don't have to keep track of two separate downloading tasks and it simplifies + * a bit so that we don't have to keep track of two separate downloading tasks and it simplifies * interfaces. If LIST_URL is passed to doInBackground(), we fetch a form list. If a hashmap * containing form/url pairs is passed, we download those forms. * @@ -51,391 +32,11 @@ */ public class DownloadFormListTask extends AsyncTask> { - // used to store error message if one occurs - public static final String DL_ERROR_MSG = "dlerrormessage"; - public static final String DL_AUTH_REQUIRED = "dlauthrequired"; - private FormListDownloaderListener stateListener; - private static final String NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_LIST = - "http://openrosa.org/xforms/xformsList"; - - - private boolean isXformsListNamespacedElement(Element e) { - return e.getNamespace().equalsIgnoreCase(NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_LIST); - } - - @Override protected HashMap doInBackground(Void... values) { - SharedPreferences settings = - PreferenceManager.getDefaultSharedPreferences( - Collect.getInstance().getBaseContext()); - String downloadListUrl = - settings.getString(PreferenceKeys.KEY_SERVER_URL, - Collect.getInstance().getString(R.string.default_server_url)); - // NOTE: /formlist must not be translated! It is the well-known path on the server. - String formListUrl = Collect.getInstance().getApplicationContext().getString( - R.string.default_odk_formlist); - String downloadPath = settings.getString(PreferenceKeys.KEY_FORMLIST_URL, formListUrl); - downloadListUrl += downloadPath; - - Collect.getInstance().getActivityLogger().logAction(this, formListUrl, downloadListUrl); - - // We populate this with available forms from the specified server. - // - HashMap formList = new HashMap(); - - // get shared HttpContext so that authentication and cookies are retained. - HttpContext localContext = Collect.getInstance().getHttpContext(); - HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); - - DocumentFetchResult result = - WebUtils.getXmlDocument(downloadListUrl, localContext, httpclient); - - // If we can't get the document, return the error, cancel the task - if (result.errorMessage != null) { - if (result.responseCode == 401) { - formList.put(DL_AUTH_REQUIRED, new FormDetails(result.errorMessage)); - } else { - formList.put(DL_ERROR_MSG, new FormDetails(result.errorMessage)); - } - return formList; - } - - if (result.isOpenRosaResponse) { - // Attempt OpenRosa 1.0 parsing - Element xformsElement = result.doc.getRootElement(); - if (!xformsElement.getName().equals("xforms")) { - String error = "root element is not : " + xformsElement.getName(); - Timber.e("Parsing OpenRosa reply -- %s", error); - formList.put( - DL_ERROR_MSG, - new FormDetails(Collect.getInstance().getString( - R.string.parse_openrosa_formlist_failed, error))); - return formList; - } - String namespace = xformsElement.getNamespace(); - if (!isXformsListNamespacedElement(xformsElement)) { - String error = "root element namespace is incorrect:" + namespace; - Timber.e("Parsing OpenRosa reply -- %s", error); - formList.put( - DL_ERROR_MSG, - new FormDetails(Collect.getInstance().getString( - R.string.parse_openrosa_formlist_failed, error))); - return formList; - } - int elements = xformsElement.getChildCount(); - for (int i = 0; i < elements; ++i) { - if (xformsElement.getType(i) != Element.ELEMENT) { - // e.g., whitespace (text) - continue; - } - Element xformElement = xformsElement.getElement(i); - if (!isXformsListNamespacedElement(xformElement)) { - // someone else's extension? - continue; - } - String name = xformElement.getName(); - if (!name.equalsIgnoreCase("xform")) { - // someone else's extension? - continue; - } - - // this is something we know how to interpret - String formId = null; - String formName = null; - String version = null; - String majorMinorVersion = null; - String description = null; - String downloadUrl = null; - String manifestUrl = null; - String hash = null; - // don't process descriptionUrl - int fieldCount = xformElement.getChildCount(); - for (int j = 0; j < fieldCount; ++j) { - if (xformElement.getType(j) != Element.ELEMENT) { - // whitespace - continue; - } - Element child = xformElement.getElement(j); - if (!isXformsListNamespacedElement(child)) { - // someone else's extension? - continue; - } - String tag = child.getName(); - switch (tag) { - case "formID": - formId = XFormParser.getXMLText(child, true); - if (formId != null && formId.length() == 0) { - formId = null; - } - break; - case "name": - formName = XFormParser.getXMLText(child, true); - if (formName != null && formName.length() == 0) { - formName = null; - } - break; - case "version": - version = XFormParser.getXMLText(child, true); - if (version != null && version.length() == 0) { - version = null; - } - break; - case "majorMinorVersion": - majorMinorVersion = XFormParser.getXMLText(child, true); - if (majorMinorVersion != null && majorMinorVersion.length() == 0) { - majorMinorVersion = null; - } - break; - case "descriptionText": - description = XFormParser.getXMLText(child, true); - if (description != null && description.length() == 0) { - description = null; - } - break; - case "downloadUrl": - downloadUrl = XFormParser.getXMLText(child, true); - if (downloadUrl != null && downloadUrl.length() == 0) { - downloadUrl = null; - } - break; - case "manifestUrl": - manifestUrl = XFormParser.getXMLText(child, true); - if (manifestUrl != null && manifestUrl.length() == 0) { - manifestUrl = null; - } - break; - case "hash": - hash = XFormParser.getXMLText(child, true); - if (hash != null && hash.length() == 0) { - hash = null; - } - break; - } - } - if (formId == null || downloadUrl == null || formName == null) { - String error = - "Forms list entry " + Integer.toString(i) - + " has missing or empty tags: formID, name, or downloadUrl"; - Timber.e("Parsing OpenRosa reply -- %s", error); - formList.clear(); - formList.put( - DL_ERROR_MSG, - new FormDetails(Collect.getInstance().getString( - R.string.parse_openrosa_formlist_failed, error))); - return formList; - } - boolean isNewerFormVersionAvailable = false; - boolean areNewerMediaFilesAvailable = false; - if (isThisFormAlreadyDownloaded(formId)) { - isNewerFormVersionAvailable = isNewerFormVersionAvailable(DownloadFormsTask.getMd5Hash(hash)); - if (!isNewerFormVersionAvailable && manifestUrl != null) { - List newMediaFiles = downloadMediaFileList(manifestUrl); - if (newMediaFiles != null && newMediaFiles.size() > 0) { - areNewerMediaFilesAvailable = areNewerMediaFilesAvailable(formId, version, newMediaFiles); - } - } - } - formList.put(formId, new FormDetails(formName, downloadUrl, manifestUrl, formId, - (version != null) ? version : majorMinorVersion, isNewerFormVersionAvailable, areNewerMediaFilesAvailable)); - } - } else { - // Aggregate 0.9.x mode... - // populate HashMap with form names and urls - Element formsElement = result.doc.getRootElement(); - int formsCount = formsElement.getChildCount(); - String formId = null; - for (int i = 0; i < formsCount; ++i) { - if (formsElement.getType(i) != Element.ELEMENT) { - // whitespace - continue; - } - Element child = formsElement.getElement(i); - String tag = child.getName(); - if (tag.equals("formID")) { - formId = XFormParser.getXMLText(child, true); - if (formId != null && formId.length() == 0) { - formId = null; - } - } - if (tag.equalsIgnoreCase("form")) { - String formName = XFormParser.getXMLText(child, true); - if (formName != null && formName.length() == 0) { - formName = null; - } - String downloadUrl = child.getAttributeValue(null, "url"); - downloadUrl = downloadUrl.trim(); - if (downloadUrl != null && downloadUrl.length() == 0) { - downloadUrl = null; - } - if (downloadUrl == null || formName == null) { - String error = - "Forms list entry " + Integer.toString(i) - + " is missing form name or url attribute"; - Timber.e("Parsing OpenRosa reply -- %s", error); - formList.clear(); - formList.put( - DL_ERROR_MSG, - new FormDetails(Collect.getInstance().getString( - R.string.parse_legacy_formlist_failed, error))); - return formList; - } - formList.put(formName, - new FormDetails(formName, downloadUrl, null, formId, null, false, false)); - - formId = null; - } - } - } - return formList; - } - - private boolean isThisFormAlreadyDownloaded(String formId) { - return new FormsDao().getFormsCursorForFormId(formId).getCount() > 0; - } - - private List downloadMediaFileList(String manifestUrl) { - if (manifestUrl == null) { - return null; - } - - // get shared HttpContext so that authentication and cookies are retained. - HttpContext localContext = Collect.getInstance().getHttpContext(); - - HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); - - DocumentFetchResult result = - WebUtils.getXmlDocument(manifestUrl, localContext, httpclient); - - if (result.errorMessage != null) { - return null; - } - - String errMessage = Collect.getInstance().getString(R.string.access_error, manifestUrl); - - if (!result.isOpenRosaResponse) { - errMessage += Collect.getInstance().getString(R.string.manifest_server_error); - Timber.e(errMessage); - return null; - } - - // Attempt OpenRosa 1.0 parsing - Element manifestElement = result.doc.getRootElement(); - if (!manifestElement.getName().equals("manifest")) { - errMessage += - Collect.getInstance().getString(R.string.root_element_error, - manifestElement.getName()); - Timber.e(errMessage); - return null; - } - String namespace = manifestElement.getNamespace(); - if (!DownloadFormsTask.isXformsManifestNamespacedElement(manifestElement)) { - errMessage += Collect.getInstance().getString(R.string.root_namespace_error, namespace); - Timber.e(errMessage); - return null; - } - int elements = manifestElement.getChildCount(); - List files = new ArrayList<>(); - for (int i = 0; i < elements; ++i) { - if (manifestElement.getType(i) != Element.ELEMENT) { - // e.g., whitespace (text) - continue; - } - Element mediaFileElement = manifestElement.getElement(i); - if (!DownloadFormsTask.isXformsManifestNamespacedElement(mediaFileElement)) { - // someone else's extension? - continue; - } - String name = mediaFileElement.getName(); - if (name.equalsIgnoreCase("mediaFile")) { - String filename = null; - String hash = null; - String downloadUrl = null; - // don't process descriptionUrl - int childCount = mediaFileElement.getChildCount(); - for (int j = 0; j < childCount; ++j) { - if (mediaFileElement.getType(j) != Element.ELEMENT) { - // e.g., whitespace (text) - continue; - } - Element child = mediaFileElement.getElement(j); - if (!DownloadFormsTask.isXformsManifestNamespacedElement(child)) { - // someone else's extension? - continue; - } - String tag = child.getName(); - switch (tag) { - case "filename": - filename = XFormParser.getXMLText(child, true); - if (filename != null && filename.length() == 0) { - filename = null; - } - break; - case "hash": - hash = XFormParser.getXMLText(child, true); - if (hash != null && hash.length() == 0) { - hash = null; - } - break; - case "downloadUrl": - downloadUrl = XFormParser.getXMLText(child, true); - if (downloadUrl != null && downloadUrl.length() == 0) { - downloadUrl = null; - } - break; - } - } - if (filename == null || downloadUrl == null || hash == null) { - errMessage += - Collect.getInstance().getString(R.string.manifest_tag_error, - Integer.toString(i)); - Timber.e(errMessage); - return null; - } - files.add(new MediaFile(filename, hash, downloadUrl)); - } - } - return files; - } - - private boolean isNewerFormVersionAvailable(String md5Hash) { - return md5Hash != null && new FormsDao().getFormsCursorForMd5Hash(md5Hash).getCount() == 0; - } - - private boolean areNewerMediaFilesAvailable(String formId, String formVersion, List newMediaFiles) { - String mediaDirPath = new FormsDao().getFormMediaPath(formId, formVersion); - if (mediaDirPath != null) { - File[] localMediaFiles = new File(mediaDirPath).listFiles(); - if (localMediaFiles != null) { - for (MediaFile newMediaFile : newMediaFiles) { - if (!isMediaFileAlreadyDownloaded(localMediaFiles, newMediaFile)) { - return true; - } - } - } else if (!newMediaFiles.isEmpty()) { - return true; - } - } - - return false; - } - - private boolean isMediaFileAlreadyDownloaded(File[] localMediaFiles, MediaFile newMediaFile) { - // TODO Zip files are ignored we should find a way to take them into account too - if (newMediaFile.getFilename().endsWith(".zip")) { - return true; - } - - String mediaFileHash = newMediaFile.getHash(); - mediaFileHash = mediaFileHash.substring(4, mediaFileHash.length()); - for (File localMediaFile : localMediaFiles) { - if (mediaFileHash.equals(FileUtils.getMd5Hash(localMediaFile))) { - return true; - } - } - return false; + return DownloadFormListUtils.downloadFormList(false); } @Override @@ -447,7 +48,6 @@ protected void onPostExecute(HashMap value) { } } - public void setDownloaderListener(FormListDownloaderListener sl) { synchronized (this) { stateListener = sl; diff --git a/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java b/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java index 01211401b47..ab8079077bc 100644 --- a/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java +++ b/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java @@ -14,48 +14,15 @@ package org.odk.collect.android.tasks; -import android.content.ContentValues; -import android.database.Cursor; -import android.net.Uri; import android.os.AsyncTask; -import org.javarosa.xform.parse.XFormParser; -import org.kxml2.kdom.Element; -import org.odk.collect.android.R; -import org.odk.collect.android.application.Collect; -import org.odk.collect.android.dao.FormsDao; +import org.odk.collect.android.listeners.DownloadFormsTaskListener; import org.odk.collect.android.listeners.FormDownloaderListener; import org.odk.collect.android.logic.FormDetails; -import org.odk.collect.android.logic.MediaFile; -import org.odk.collect.android.provider.FormsProviderAPI.FormsColumns; -import org.odk.collect.android.utilities.DocumentFetchResult; -import org.odk.collect.android.utilities.FileUtils; -import org.odk.collect.android.utilities.Validator; -import org.odk.collect.android.utilities.WebUtils; -import org.opendatakit.httpclientandroidlib.Header; -import org.opendatakit.httpclientandroidlib.HttpEntity; -import org.opendatakit.httpclientandroidlib.HttpResponse; -import org.opendatakit.httpclientandroidlib.HttpStatus; -import org.opendatakit.httpclientandroidlib.client.HttpClient; -import org.opendatakit.httpclientandroidlib.client.methods.HttpGet; -import org.opendatakit.httpclientandroidlib.protocol.HttpContext; +import org.odk.collect.android.utilities.FormDownloader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.zip.GZIPInputStream; - -import timber.log.Timber; /** * Background task for downloading a given list of forms. We assume right now that the forms are @@ -66,59 +33,25 @@ * @author carlhartung */ public class DownloadFormsTask extends - AsyncTask, String, HashMap> { - - private static final String MD5_COLON_PREFIX = "md5:"; - private static final String TEMP_DOWNLOAD_EXTENSION = ".tempDownload"; - - private FormDownloaderListener stateListener; + AsyncTask, String, HashMap> implements FormDownloaderListener { - private FormsDao formsDao; + private DownloadFormsTaskListener stateListener; - private static final String NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_MANIFEST = - "http://openrosa.org/xforms/xformsManifest"; - - static boolean isXformsManifestNamespacedElement(Element e) { - return e.getNamespace().equalsIgnoreCase(NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_MANIFEST); + @Override + public void progressUpdate(String currentFile, String progress, String total) { + publishProgress(currentFile, progress, total); } - private class TaskCancelledException extends Exception { - private final File file; - - TaskCancelledException(File file) { - super("Task was cancelled during processing of " + file); - this.file = file; - } - - TaskCancelledException() { - super("Task was cancelled"); - this.file = null; - } + @Override + public boolean isTaskCanceled() { + return isCancelled(); } @Override protected HashMap doInBackground(ArrayList... values) { - ArrayList toDownload = values[0]; - - formsDao = new FormsDao(); - int total = toDownload.size(); - int count = 1; - Collect.getInstance().getActivityLogger().logAction(this, "downloadForms", - String.valueOf(total)); - - final HashMap result = new HashMap<>(); - - for (FormDetails fd : toDownload) { - try { - String message = processOneForm(total, count++, fd); - result.put(fd, message.isEmpty() ? - Collect.getInstance().getString(R.string.success) : message); - } catch (TaskCancelledException cd) { - break; - } - } - - return result; + FormDownloader formDownloader = new FormDownloader(); + formDownloader.setDownloaderListener(this); + return formDownloader.downloadForms(values[0]); } @Override @@ -130,607 +63,6 @@ protected void onCancelled(HashMap formDetailsStringHashMap } } - /** - * Processes one form download. - * - * @param total the total number of forms being downloaded by this task - * @param count the number of this form - * @param fd the FormDetails - * @return an empty string for success, or a nonblank string with one or more error messages - * @throws TaskCancelledException to signal that form downloading is to be canceled - */ - private String processOneForm(int total, int count, FormDetails fd) throws TaskCancelledException { - publishProgress(fd.getFormName(), String.valueOf(count), String.valueOf(total)); - String message = ""; - if (isCancelled()) { - throw new TaskCancelledException(); - } - - String tempMediaPath = null; - final String finalMediaPath; - FileResult fileResult = null; - try { - // get the xml file - // if we've downloaded a duplicate, this gives us the file - fileResult = downloadXform(fd.getFormName(), fd.getDownloadUrl()); - - if (fd.getManifestUrl() != null) { - // use a temporary media path until everything is ok. - tempMediaPath = new File(Collect.CACHE_PATH, - String.valueOf(System.currentTimeMillis())).getAbsolutePath(); - finalMediaPath = FileUtils.constructMediaPath( - fileResult.getFile().getAbsolutePath()); - String error = downloadManifestAndMediaFiles(tempMediaPath, finalMediaPath, fd, - count, total); - if (error != null) { - message += error; - } - } else { - Timber.i("No Manifest for: %s", fd.getFormName()); - } - } catch (TaskCancelledException e) { - Timber.i(e); - cleanUp(fileResult, e.file, tempMediaPath); - - // do not download additional forms. - throw e; - } catch (Exception e) { - message += getExceptionMessage(e); - } - - if (isCancelled()) { - cleanUp(fileResult, null, tempMediaPath); - fileResult = null; - } - - Map parsedFields = null; - if (fileResult != null) { - try { - final long start = System.currentTimeMillis(); - Timber.w("Parsing document %s", fileResult.file.getAbsolutePath()); - parsedFields = FileUtils.parseXML(fileResult.file); - Timber.i("Parse finished in %.3f seconds.", - (System.currentTimeMillis() - start) / 1000F); - } catch (RuntimeException e) { - message += e.getMessage(); - } - } - - boolean installed = false; - - if (!isCancelled() && message.isEmpty() && parsedFields != null) { - if (isSubmissionOk(parsedFields)) { - installEverything(tempMediaPath, fileResult, parsedFields); - installed = true; - } else { - message += Collect.getInstance().getString(R.string.xform_parse_error, - fileResult.file.getName(), "submission url"); - } - } - if (!installed) { - cleanUp(fileResult, null, tempMediaPath); - } - return message; - } - - private boolean isSubmissionOk(Map parsedFields) { - String submission = parsedFields.get(FileUtils.SUBMISSIONURI); - return submission == null || Validator.isUrlValid(submission); - } - - private void installEverything(String tempMediaPath, FileResult fileResult, Map parsedFields) { - UriResult uriResult = null; - try { - uriResult = findExistingOrCreateNewUri(fileResult.file, parsedFields); - Timber.w("Form uri = %s, isNew = %b", uriResult.getUri().toString(), uriResult.isNew()); - - // move the media files in the media folder - if (tempMediaPath != null) { - File formMediaPath = new File(uriResult.getMediaPath()); - FileUtils.moveMediaFiles(tempMediaPath, formMediaPath); - } - } catch (IOException e) { - Timber.e(e); - - if (uriResult != null && uriResult.isNew() && fileResult.isNew()) { - // this means we should delete the entire form together with the metadata - Uri uri = uriResult.getUri(); - Timber.w("The form is new. We should delete the entire form."); - int deletedCount = Collect.getInstance().getContentResolver().delete(uri, - null, null); - Timber.w("Deleted %d rows using uri %s", deletedCount, uri.toString()); - } - - cleanUp(fileResult, null, tempMediaPath); - } - } - - private void cleanUp(FileResult fileResult, File fileOnCancel, String tempMediaPath) { - if (fileResult == null) { - Timber.w("The user cancelled (or an exception happened) the download of a form at the " - + "very beginning."); - } else { - formsDao.deleteFormsFromMd5Hash(FileUtils.getMd5Hash(fileResult.file)); - FileUtils.deleteAndReport(fileResult.getFile()); - } - - FileUtils.deleteAndReport(fileOnCancel); - - if (tempMediaPath != null) { - FileUtils.purgeMediaPath(tempMediaPath); - } - } - - private String getExceptionMessage(Exception e) { - String msg = e.getMessage(); - if (msg == null) { - msg = e.toString(); - } - Timber.e(msg); - - if (e.getCause() != null) { - msg = e.getCause().getMessage(); - if (msg == null) { - msg = e.getCause().toString(); - } - } - return msg; - } - - /** - * Creates a new form in the database, if none exists with the same absolute path. Returns - * information with the URI, media path, and whether the form is new. - * - * @param formFile the form definition file - * @param formInfo certain fields extracted from the parsed XML form, such as title and form ID - * @return a {@link org.odk.collect.android.tasks.DownloadFormsTask.UriResult} object - */ - private UriResult findExistingOrCreateNewUri(File formFile, Map formInfo) { - Cursor cursor = null; - final Uri uri; - final String formFilePath = formFile.getAbsolutePath(); - String mediaPath = FileUtils.constructMediaPath(formFilePath); - final boolean isNew; - - FileUtils.checkMediaPath(new File(mediaPath)); - - try { - cursor = formsDao.getFormsCursorForFormFilePath(formFile.getAbsolutePath()); - isNew = cursor.getCount() <= 0; - - if (isNew) { - uri = saveNewForm(formInfo, formFile, mediaPath); - } else { - cursor.moveToFirst(); - uri = Uri.withAppendedPath(FormsColumns.CONTENT_URI, - cursor.getString(cursor.getColumnIndex(FormsColumns._ID))); - mediaPath = cursor.getString(cursor.getColumnIndex(FormsColumns.FORM_MEDIA_PATH)); - Collect.getInstance().getActivityLogger().logAction(this, "refresh", - formFilePath); - } - } finally { - if (cursor != null) { - cursor.close(); - } - } - - return new UriResult(uri, mediaPath, isNew); - } - - private Uri saveNewForm(Map formInfo, File formFile, String mediaPath) { - final ContentValues v = new ContentValues(); - v.put(FormsColumns.FORM_FILE_PATH, formFile.getAbsolutePath()); - v.put(FormsColumns.FORM_MEDIA_PATH, mediaPath); - v.put(FormsColumns.DISPLAY_NAME, formInfo.get(FileUtils.TITLE)); - v.put(FormsColumns.JR_VERSION, formInfo.get(FileUtils.VERSION)); - v.put(FormsColumns.JR_FORM_ID, formInfo.get(FileUtils.FORMID)); - v.put(FormsColumns.SUBMISSION_URI, formInfo.get(FileUtils.SUBMISSIONURI)); - v.put(FormsColumns.BASE64_RSA_PUBLIC_KEY, formInfo.get(FileUtils.BASE64_RSA_PUBLIC_KEY)); - v.put(FormsColumns.AUTO_DELETE, formInfo.get(FileUtils.AUTO_DELETE)); - v.put(FormsColumns.AUTO_SEND, formInfo.get(FileUtils.AUTO_SEND)); - Uri uri = formsDao.saveForm(v); - Collect.getInstance().getActivityLogger().logAction(this, "insert", - formFile.getAbsolutePath()); - return uri; - } - - /** - * Takes the formName and the URL and attempts to download the specified file. Returns a file - * object representing the downloaded file. - */ - private FileResult downloadXform(String formName, String url) - throws IOException, TaskCancelledException, Exception { - // clean up friendly form name... - String rootName = formName.replaceAll("[^\\p{L}\\p{Digit}]", " "); - rootName = rootName.replaceAll("\\p{javaWhitespace}+", " "); - rootName = rootName.trim(); - - // proposed name of xml file... - String path = Collect.FORMS_PATH + File.separator + rootName + ".xml"; - int i = 2; - File f = new File(path); - while (f.exists()) { - path = Collect.FORMS_PATH + File.separator + rootName + "_" + i + ".xml"; - f = new File(path); - i++; - } - - downloadFile(f, url); - - boolean isNew = true; - - // we've downloaded the file, and we may have renamed it - // make sure it's not the same as a file we already have - Cursor c = null; - try { - c = formsDao.getFormsCursorForMd5Hash(FileUtils.getMd5Hash(f)); - if (c.getCount() > 0) { - // Should be at most, 1 - c.moveToFirst(); - - isNew = false; - - // delete the file we just downloaded, because it's a duplicate - Timber.w("A duplicate file has been found, we need to remove the downloaded file " - + "and return the other one."); - FileUtils.deleteAndReport(f); - - // set the file returned to the file we already had - String existingPath = c.getString(c.getColumnIndex(FormsColumns.FORM_FILE_PATH)); - f = new File(existingPath); - Timber.w("Will use %s", existingPath); - } - } finally { - if (c != null) { - c.close(); - } - } - - return new FileResult(f, isNew); - } - - /** - * Common routine to download a document from the downloadUrl and save the contents in the file - * 'file'. Shared by media file download and form file download. - *

- * SurveyCTO: The file is saved into a temp folder and is moved to the final place if everything - * is okay, so that garbage is not left over on cancel. - * - * @param file the final file - * @param downloadUrl the url to get the contents from. - */ - private void downloadFile(File file, String downloadUrl) - throws IOException, TaskCancelledException, URISyntaxException, Exception { - File tempFile = File.createTempFile(file.getName(), TEMP_DOWNLOAD_EXTENSION, - new File(Collect.CACHE_PATH)); - - URI uri; - try { - // assume the downloadUrl is escaped properly - URL url = new URL(downloadUrl); - uri = url.toURI(); - } catch (MalformedURLException | URISyntaxException e) { - Timber.e(e, "Unable to get a URI for download URL : %s due to %s : ", downloadUrl, e.getMessage()); - throw e; - } - - // WiFi network connections can be renegotiated during a large form download sequence. - // This will cause intermittent download failures. Silently retry once after each - // failure. Only if there are two consecutive failures do we abort. - boolean success = false; - int attemptCount = 0; - final int MAX_ATTEMPT_COUNT = 2; - while (!success && ++attemptCount <= MAX_ATTEMPT_COUNT) { - if (isCancelled()) { - throw new TaskCancelledException(tempFile); - } - Timber.i("Started downloading to %s from %s", tempFile.getAbsolutePath(), downloadUrl); - - // get shared HttpContext so that authentication and cookies are retained. - HttpContext localContext = Collect.getInstance().getHttpContext(); - - HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); - - // set up request... - HttpGet req = WebUtils.createOpenRosaHttpGet(uri); - req.addHeader(WebUtils.ACCEPT_ENCODING_HEADER, WebUtils.GZIP_CONTENT_ENCODING); - - HttpResponse response; - try { - response = httpclient.execute(req, localContext); - int statusCode = response.getStatusLine().getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - WebUtils.discardEntityBytes(response); - if (statusCode == HttpStatus.SC_UNAUTHORIZED) { - // clear the cookies -- should not be necessary? - Collect.getInstance().getCookieStore().clear(); - } - String errMsg = - Collect.getInstance().getString(R.string.file_fetch_failed, downloadUrl, - response.getStatusLine().getReasonPhrase(), String.valueOf(statusCode)); - Timber.e(errMsg); - throw new Exception(errMsg); - } - - // write connection to file - InputStream is = null; - OutputStream os = null; - try { - HttpEntity entity = response.getEntity(); - is = entity.getContent(); - Header contentEncoding = entity.getContentEncoding(); - if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase( - WebUtils.GZIP_CONTENT_ENCODING)) { - is = new GZIPInputStream(is); - } - os = new FileOutputStream(tempFile); - byte[] buf = new byte[4096]; - int len; - while ((len = is.read(buf)) > 0 && !isCancelled()) { - os.write(buf, 0, len); - } - os.flush(); - success = true; - } finally { - if (os != null) { - try { - os.close(); - } catch (Exception e) { - Timber.e(e); - } - } - if (is != null) { - try { - // ensure stream is consumed... - final long count = 1024L; - while (is.skip(count) == count) { - // skipping to the end of the http entity - } - } catch (Exception e) { - // no-op - } - try { - is.close(); - } catch (Exception e) { - Timber.e(e); - } - } - } - } catch (Exception e) { - Timber.e(e.toString()); - // silently retry unless this is the last attempt, - // in which case we rethrow the exception. - - FileUtils.deleteAndReport(tempFile); - - if (attemptCount == MAX_ATTEMPT_COUNT) { - throw e; - } - } - - if (isCancelled()) { - FileUtils.deleteAndReport(tempFile); - throw new TaskCancelledException(tempFile); - } - } - - Timber.d("Completed downloading of %s. It will be moved to the proper path...", - tempFile.getAbsolutePath()); - - FileUtils.deleteAndReport(file); - - String errorMessage = FileUtils.copyFile(tempFile, file); - - if (file.exists()) { - Timber.w("Copied %s over %s", tempFile.getAbsolutePath(), file.getAbsolutePath()); - FileUtils.deleteAndReport(tempFile); - } else { - String msg = Collect.getInstance().getString(R.string.fs_file_copy_error, - tempFile.getAbsolutePath(), file.getAbsolutePath(), errorMessage); - Timber.w(msg); - throw new RuntimeException(msg); - } - } - - private static class UriResult { - - private final Uri uri; - private final String mediaPath; - private final boolean isNew; - - private UriResult(Uri uri, String mediaPath, boolean isNew) { - this.uri = uri; - this.mediaPath = mediaPath; - this.isNew = isNew; - } - - private Uri getUri() { - return uri; - } - - private String getMediaPath() { - return mediaPath; - } - - private boolean isNew() { - return isNew; - } - } - - private static class FileResult { - - private final File file; - private final boolean isNew; - - private FileResult(File file, boolean isNew) { - this.file = file; - this.isNew = isNew; - } - - private File getFile() { - return file; - } - - private boolean isNew() { - return isNew; - } - } - - private String downloadManifestAndMediaFiles(String tempMediaPath, String finalMediaPath, - FormDetails fd, int count, - int total) throws Exception { - if (fd.getManifestUrl() == null) { - return null; - } - - publishProgress(Collect.getInstance().getString(R.string.fetching_manifest, fd.getFormName()), - String.valueOf(count), String.valueOf(total)); - - List files = new ArrayList(); - // get shared HttpContext so that authentication and cookies are retained. - HttpContext localContext = Collect.getInstance().getHttpContext(); - - HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); - - DocumentFetchResult result = - WebUtils.getXmlDocument(fd.getManifestUrl(), localContext, httpclient); - - if (result.errorMessage != null) { - return result.errorMessage; - } - - String errMessage = Collect.getInstance().getString(R.string.access_error, fd.getManifestUrl()); - - if (!result.isOpenRosaResponse) { - errMessage += Collect.getInstance().getString(R.string.manifest_server_error); - Timber.e(errMessage); - return errMessage; - } - - // Attempt OpenRosa 1.0 parsing - Element manifestElement = result.doc.getRootElement(); - if (!manifestElement.getName().equals("manifest")) { - errMessage += - Collect.getInstance().getString(R.string.root_element_error, - manifestElement.getName()); - Timber.e(errMessage); - return errMessage; - } - String namespace = manifestElement.getNamespace(); - if (!isXformsManifestNamespacedElement(manifestElement)) { - errMessage += Collect.getInstance().getString(R.string.root_namespace_error, namespace); - Timber.e(errMessage); - return errMessage; - } - int elements = manifestElement.getChildCount(); - for (int i = 0; i < elements; ++i) { - if (manifestElement.getType(i) != Element.ELEMENT) { - // e.g., whitespace (text) - continue; - } - Element mediaFileElement = manifestElement.getElement(i); - if (!isXformsManifestNamespacedElement(mediaFileElement)) { - // someone else's extension? - continue; - } - String name = mediaFileElement.getName(); - if (name.equalsIgnoreCase("mediaFile")) { - String filename = null; - String hash = null; - String downloadUrl = null; - // don't process descriptionUrl - int childCount = mediaFileElement.getChildCount(); - for (int j = 0; j < childCount; ++j) { - if (mediaFileElement.getType(j) != Element.ELEMENT) { - // e.g., whitespace (text) - continue; - } - Element child = mediaFileElement.getElement(j); - if (!isXformsManifestNamespacedElement(child)) { - // someone else's extension? - continue; - } - String tag = child.getName(); - switch (tag) { - case "filename": - filename = XFormParser.getXMLText(child, true); - if (filename != null && filename.length() == 0) { - filename = null; - } - break; - case "hash": - hash = XFormParser.getXMLText(child, true); - if (hash != null && hash.length() == 0) { - hash = null; - } - break; - case "downloadUrl": - downloadUrl = XFormParser.getXMLText(child, true); - if (downloadUrl != null && downloadUrl.length() == 0) { - downloadUrl = null; - } - break; - } - } - if (filename == null || downloadUrl == null || hash == null) { - errMessage += - Collect.getInstance().getString(R.string.manifest_tag_error, - Integer.toString(i)); - Timber.e(errMessage); - return errMessage; - } - files.add(new MediaFile(filename, hash, downloadUrl)); - } - } - - // OK we now have the full set of files to download... - Timber.i("Downloading %d media files.", files.size()); - int mediaCount = 0; - if (files.size() > 0) { - File tempMediaDir = new File(tempMediaPath); - File finalMediaDir = new File(finalMediaPath); - - FileUtils.checkMediaPath(tempMediaDir); - FileUtils.checkMediaPath(finalMediaDir); - - for (MediaFile toDownload : files) { - ++mediaCount; - publishProgress( - Collect.getInstance().getString(R.string.form_download_progress, - fd.getFormName(), - String.valueOf(mediaCount), String.valueOf(files.size())), - String.valueOf(count), String.valueOf(total)); - //try { - File finalMediaFile = new File(finalMediaDir, toDownload.getFilename()); - File tempMediaFile = new File(tempMediaDir, toDownload.getFilename()); - - if (!finalMediaFile.exists()) { - downloadFile(tempMediaFile, toDownload.getDownloadUrl()); - } else { - String currentFileHash = FileUtils.getMd5Hash(finalMediaFile); - String downloadFileHash = getMd5Hash(toDownload.getHash()); - - if (currentFileHash != null && downloadFileHash != null && !currentFileHash.contentEquals(downloadFileHash)) { - // if the hashes match, it's the same file - // otherwise delete our current one and replace it with the new one - FileUtils.deleteAndReport(finalMediaFile); - downloadFile(tempMediaFile, toDownload.getDownloadUrl()); - } else { - // exists, and the hash is the same - // no need to download it again - Timber.i("Skipping media file fetch -- file hashes identical: %s", - finalMediaFile.getAbsolutePath()); - } - } - // } catch (Exception e) { - // return e.getLocalizedMessage(); - //} - } - } - return null; - } - @Override protected void onPostExecute(HashMap value) { synchronized (this) { @@ -753,13 +85,9 @@ protected void onProgressUpdate(String... values) { } - public void setDownloaderListener(FormDownloaderListener sl) { + public void setDownloaderListener(DownloadFormsTaskListener sl) { synchronized (this) { stateListener = sl; } } - - static String getMd5Hash(String hash) { - return hash == null ? null : hash.substring(MD5_COLON_PREFIX.length()); - } } diff --git a/collect_app/src/main/java/org/odk/collect/android/tasks/ServerPollingJob.java b/collect_app/src/main/java/org/odk/collect/android/tasks/ServerPollingJob.java new file mode 100644 index 00000000000..17d1f1d3bde --- /dev/null +++ b/collect_app/src/main/java/org/odk/collect/android/tasks/ServerPollingJob.java @@ -0,0 +1,204 @@ +/* + * Copyright 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.odk.collect.android.tasks; + +import android.app.NotificationManager; +import android.app.PendingIntent; +import android.content.ContentValues; +import android.content.Context; +import android.content.Intent; +import android.database.Cursor; +import android.net.ConnectivityManager; +import android.net.NetworkInfo; +import android.support.annotation.NonNull; +import android.support.v4.app.NotificationCompat; + +import com.evernote.android.job.Job; +import com.evernote.android.job.JobManager; +import com.evernote.android.job.JobRequest; + +import org.odk.collect.android.R; +import org.odk.collect.android.activities.FormDownloadList; +import org.odk.collect.android.activities.NotificationActivity; +import org.odk.collect.android.application.Collect; +import org.odk.collect.android.dao.FormsDao; +import org.odk.collect.android.logic.FormDetails; +import org.odk.collect.android.preferences.GeneralSharedPreferences; +import org.odk.collect.android.utilities.AuthDialogUtility; +import org.odk.collect.android.utilities.DownloadFormListUtils; +import org.odk.collect.android.utilities.FormDownloader; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.odk.collect.android.activities.FormDownloadList.DISPLAY_ONLY_UPDATED_FORMS; +import static org.odk.collect.android.preferences.PreferenceKeys.KEY_AUTOMATIC_UPDATE; +import static org.odk.collect.android.preferences.PreferenceKeys.KEY_PERIODIC_FORM_UPDATES_CHECK; +import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.JR_FORM_ID; +import static org.odk.collect.android.provider.FormsProviderAPI.FormsColumns.LAST_DETECTED_FORM_VERSION_HASH; +import static org.odk.collect.android.utilities.DownloadFormListUtils.DL_AUTH_REQUIRED; +import static org.odk.collect.android.utilities.DownloadFormListUtils.DL_ERROR_MSG; + +public class ServerPollingJob extends Job { + + private static final long FIFTEEN_MINUTES_PERIOD = 900000; + private static final long ONE_HOUR_PERIOD = 3600000; + private static final long SIX_HOURS_PERIOD = 21600000; + private static final long ONE_DAY_PERIOD = 86400000; + + private static final String POLL_SERVER_IMMEDIATELY_AFTER_RECEIVING_NETWORK = "pollServerImmediatelyAfterReceivingNetwork"; + public static final String TAG = "serverPollingJob"; + + @Override + @NonNull + protected Result onRunJob(@NonNull Params params) { + if (!isDeviceOnline()) { + GeneralSharedPreferences.getInstance().save(POLL_SERVER_IMMEDIATELY_AFTER_RECEIVING_NETWORK, true); + return Result.FAILURE; + } else { + GeneralSharedPreferences.getInstance().reset(POLL_SERVER_IMMEDIATELY_AFTER_RECEIVING_NETWORK); + HashMap formList = DownloadFormListUtils.downloadFormList(true); + + if (formList != null && !formList.containsKey(DL_ERROR_MSG)) { + if (formList.containsKey(DL_AUTH_REQUIRED)) { + AuthDialogUtility.setWebCredentialsFromPreferences(); + formList = DownloadFormListUtils.downloadFormList(true); + + if (formList == null || formList.containsKey(DL_AUTH_REQUIRED) || formList.containsKey(DL_ERROR_MSG)) { + return Result.FAILURE; + } + } + + List newDetectedForms = new ArrayList<>(); + for (FormDetails formDetails : formList.values()) { + if (formDetails.isNewerFormVersionAvailable() || formDetails.areNewerMediaFilesAvailable()) { + newDetectedForms.add(formDetails); + } + } + + if (!newDetectedForms.isEmpty()) { + if (GeneralSharedPreferences.getInstance().getBoolean(KEY_AUTOMATIC_UPDATE, false)) { + final HashMap result = new FormDownloader().downloadForms(newDetectedForms); + informAboutNewDownloadedForms(Collect.getInstance().getString(R.string.forms_downloaded) + "\n\n" + FormDownloadList.getDownloadResultMessage(result)); + } else { + for (FormDetails formDetails : newDetectedForms) { + String manifestFileHash = formDetails.getManifestFileHash() != null ? formDetails.getManifestFileHash() : ""; + String formVersionHash = FormDownloader.getMd5Hash(formDetails.getHash()) + manifestFileHash; + if (!wasThisNewerFormVersionAlreadyDetected(formVersionHash)) { + updateLastDetectedFormVersionHash(formDetails.getFormID(), formVersionHash); + } else { + newDetectedForms.remove(formDetails); + } + } + + if (!newDetectedForms.isEmpty()) { + informAboutNewAvailableForms(); + } + } + } + return Result.SUCCESS; + } else { + return Result.FAILURE; + } + } + } + + public static void schedulePeriodicJob(String selectedOption) { + if (selectedOption.equals(Collect.getInstance().getString(R.string.never_value))) { + JobManager.instance().cancelAllForTag(TAG); + GeneralSharedPreferences.getInstance().reset(POLL_SERVER_IMMEDIATELY_AFTER_RECEIVING_NETWORK); + } else { + long period = FIFTEEN_MINUTES_PERIOD; + if (selectedOption.equals(Collect.getInstance().getString(R.string.every_one_hour_value))) { + period = ONE_HOUR_PERIOD; + } else if (selectedOption.equals(Collect.getInstance().getString(R.string.every_six_hours_value))) { + period = SIX_HOURS_PERIOD; + } else if (selectedOption.equals(Collect.getInstance().getString(R.string.every_24_hours_value))) { + period = ONE_DAY_PERIOD; + } + + new JobRequest.Builder(TAG) + .setPeriodic(period, 300000) + .setUpdateCurrent(true) + .build() + .schedule(); + } + } + + private boolean wasThisNewerFormVersionAlreadyDetected(String formVersionHash) { + Cursor cursor = new FormsDao().getFormsCursor(LAST_DETECTED_FORM_VERSION_HASH + "=?", new String[]{formVersionHash}); + return cursor == null || cursor.getCount() > 0; + } + + private void informAboutNewAvailableForms() { + Intent intent = new Intent(getContext(), FormDownloadList.class); + intent.putExtra(DISPLAY_ONLY_UPDATED_FORMS, true); + PendingIntent contentIntent = PendingIntent.getActivity(getContext(), 0, intent, PendingIntent.FLAG_UPDATE_CURRENT); + + NotificationCompat.Builder builder = new NotificationCompat.Builder(getContext()) + .setSmallIcon(R.drawable.notes) + .setContentTitle(getContext().getString(R.string.form_updates_available)) + .setAutoCancel(true) + .setContentIntent(contentIntent); + + NotificationManager manager = (NotificationManager) getContext().getSystemService(Context.NOTIFICATION_SERVICE); + if (manager != null) { + manager.notify(0, builder.build()); + } + } + + private void informAboutNewDownloadedForms(String message) { + Intent intent = new Intent(Collect.getInstance(), NotificationActivity.class); + intent.putExtra(NotificationActivity.NOTIFICATION_KEY, message); + PendingIntent contentIntent = PendingIntent.getActivity(getContext(), 0, intent, PendingIntent.FLAG_UPDATE_CURRENT); + + NotificationCompat.Builder builder = new NotificationCompat.Builder(getContext()) + .setSmallIcon(R.drawable.notes) + .setContentTitle(getContext().getString(R.string.new_form_versions_downloaded)) + .setAutoCancel(true) + .setContentIntent(contentIntent); + + NotificationManager manager = (NotificationManager) getContext().getSystemService(Context.NOTIFICATION_SERVICE); + if (manager != null) { + manager.notify(0, builder.build()); + } + } + + private void updateLastDetectedFormVersionHash(String formId, String formVersionHash) { + ContentValues values = new ContentValues(); + values.put(LAST_DETECTED_FORM_VERSION_HASH, formVersionHash); + new FormsDao().updateForm(values, JR_FORM_ID + "=?", new String[] {formId}); + } + + private boolean isDeviceOnline() { + ConnectivityManager connMgr = + (ConnectivityManager) Collect.getInstance().getSystemService(Context.CONNECTIVITY_SERVICE); + NetworkInfo networkInfo = connMgr.getActiveNetworkInfo(); + return (networkInfo != null && networkInfo.isConnected()); + } + + public static void pollServerIfNeeded() { + if (GeneralSharedPreferences.getInstance().getBoolean(POLL_SERVER_IMMEDIATELY_AFTER_RECEIVING_NETWORK, false) + && !GeneralSharedPreferences.getInstance().get(KEY_PERIODIC_FORM_UPDATES_CHECK).equals(Collect.getInstance().getString(R.string.never_value))) { + new JobRequest.Builder(TAG) + .startNow() + .build() + .schedule(); + } + } +} \ No newline at end of file diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/DocumentFetchResult.java b/collect_app/src/main/java/org/odk/collect/android/utilities/DocumentFetchResult.java index bbd8aadde41..eb7e21f961f 100644 --- a/collect_app/src/main/java/org/odk/collect/android/utilities/DocumentFetchResult.java +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/DocumentFetchResult.java @@ -21,6 +21,7 @@ public class DocumentFetchResult { public final int responseCode; public final Document doc; public final boolean isOpenRosaResponse; + private String hash; public DocumentFetchResult(String msg, int response) { @@ -31,10 +32,15 @@ public DocumentFetchResult(String msg, int response) { } - public DocumentFetchResult(Document doc, boolean isOpenRosaResponse) { + public DocumentFetchResult(Document doc, boolean isOpenRosaResponse, String hash) { responseCode = 0; errorMessage = null; this.doc = doc; this.isOpenRosaResponse = isOpenRosaResponse; + this.hash = hash; + } + + public String getHash() { + return hash; } } \ No newline at end of file diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/DownloadFormListUtils.java b/collect_app/src/main/java/org/odk/collect/android/utilities/DownloadFormListUtils.java new file mode 100644 index 00000000000..26c366e83f8 --- /dev/null +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/DownloadFormListUtils.java @@ -0,0 +1,432 @@ +/* + * Copyright 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.odk.collect.android.utilities; + +import android.content.SharedPreferences; +import android.preference.PreferenceManager; + +import org.javarosa.xform.parse.XFormParser; +import org.kxml2.kdom.Element; +import org.odk.collect.android.R; +import org.odk.collect.android.application.Collect; +import org.odk.collect.android.dao.FormsDao; +import org.odk.collect.android.logic.FormDetails; +import org.odk.collect.android.logic.ManifestFile; +import org.odk.collect.android.logic.MediaFile; +import org.odk.collect.android.preferences.PreferenceKeys; +import org.opendatakit.httpclientandroidlib.client.HttpClient; +import org.opendatakit.httpclientandroidlib.protocol.HttpContext; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import timber.log.Timber; + +public class DownloadFormListUtils { + + // used to store error message if one occurs + public static final String DL_ERROR_MSG = "dlerrormessage"; + public static final String DL_AUTH_REQUIRED = "dlauthrequired"; + + private static final String NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_LIST = + "http://openrosa.org/xforms/xformsList"; + + private static boolean isXformsListNamespacedElement(Element e) { + return e.getNamespace().equalsIgnoreCase(NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_LIST); + } + + private DownloadFormListUtils() { + } + + public static HashMap downloadFormList(boolean alwaysCheckMediaFiles) { + SharedPreferences settings = + PreferenceManager.getDefaultSharedPreferences( + Collect.getInstance().getBaseContext()); + String downloadListUrl = + settings.getString(PreferenceKeys.KEY_SERVER_URL, + Collect.getInstance().getString(R.string.default_server_url)); + // NOTE: /formlist must not be translated! It is the well-known path on the server. + String formListUrl = Collect.getInstance().getApplicationContext().getString( + R.string.default_odk_formlist); + String downloadPath = settings.getString(PreferenceKeys.KEY_FORMLIST_URL, formListUrl); + downloadListUrl += downloadPath; + + // We populate this with available forms from the specified server. + // + HashMap formList = new HashMap(); + + // get shared HttpContext so that authentication and cookies are retained. + HttpContext localContext = Collect.getInstance().getHttpContext(); + HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); + + DocumentFetchResult result = + WebUtils.getXmlDocument(downloadListUrl, localContext, httpclient); + + // If we can't get the document, return the error, cancel the task + if (result.errorMessage != null) { + if (result.responseCode == 401) { + formList.put(DL_AUTH_REQUIRED, new FormDetails(result.errorMessage)); + } else { + formList.put(DL_ERROR_MSG, new FormDetails(result.errorMessage)); + } + return formList; + } + + if (result.isOpenRosaResponse) { + // Attempt OpenRosa 1.0 parsing + Element xformsElement = result.doc.getRootElement(); + if (!xformsElement.getName().equals("xforms")) { + String error = "root element is not : " + xformsElement.getName(); + Timber.e("Parsing OpenRosa reply -- %s", error); + formList.put( + DL_ERROR_MSG, + new FormDetails(Collect.getInstance().getString( + R.string.parse_openrosa_formlist_failed, error))); + return formList; + } + String namespace = xformsElement.getNamespace(); + if (!isXformsListNamespacedElement(xformsElement)) { + String error = "root element namespace is incorrect:" + namespace; + Timber.e("Parsing OpenRosa reply -- %s", error); + formList.put( + DL_ERROR_MSG, + new FormDetails(Collect.getInstance().getString( + R.string.parse_openrosa_formlist_failed, error))); + return formList; + } + int elements = xformsElement.getChildCount(); + for (int i = 0; i < elements; ++i) { + if (xformsElement.getType(i) != Element.ELEMENT) { + // e.g., whitespace (text) + continue; + } + Element xformElement = xformsElement.getElement(i); + if (!isXformsListNamespacedElement(xformElement)) { + // someone else's extension? + continue; + } + String name = xformElement.getName(); + if (!name.equalsIgnoreCase("xform")) { + // someone else's extension? + continue; + } + + // this is something we know how to interpret + String formId = null; + String formName = null; + String version = null; + String majorMinorVersion = null; + String description = null; + String downloadUrl = null; + String manifestUrl = null; + String hash = null; + // don't process descriptionUrl + int fieldCount = xformElement.getChildCount(); + for (int j = 0; j < fieldCount; ++j) { + if (xformElement.getType(j) != Element.ELEMENT) { + // whitespace + continue; + } + Element child = xformElement.getElement(j); + if (!isXformsListNamespacedElement(child)) { + // someone else's extension? + continue; + } + String tag = child.getName(); + switch (tag) { + case "formID": + formId = XFormParser.getXMLText(child, true); + if (formId != null && formId.length() == 0) { + formId = null; + } + break; + case "name": + formName = XFormParser.getXMLText(child, true); + if (formName != null && formName.length() == 0) { + formName = null; + } + break; + case "version": + version = XFormParser.getXMLText(child, true); + if (version != null && version.length() == 0) { + version = null; + } + break; + case "majorMinorVersion": + majorMinorVersion = XFormParser.getXMLText(child, true); + if (majorMinorVersion != null && majorMinorVersion.length() == 0) { + majorMinorVersion = null; + } + break; + case "descriptionText": + description = XFormParser.getXMLText(child, true); + if (description != null && description.length() == 0) { + description = null; + } + break; + case "downloadUrl": + downloadUrl = XFormParser.getXMLText(child, true); + if (downloadUrl != null && downloadUrl.length() == 0) { + downloadUrl = null; + } + break; + case "manifestUrl": + manifestUrl = XFormParser.getXMLText(child, true); + if (manifestUrl != null && manifestUrl.length() == 0) { + manifestUrl = null; + } + break; + case "hash": + hash = XFormParser.getXMLText(child, true); + if (hash != null && hash.length() == 0) { + hash = null; + } + break; + } + } + if (formId == null || downloadUrl == null || formName == null) { + String error = + "Forms list entry " + Integer.toString(i) + + " has missing or empty tags: formID, name, or downloadUrl"; + Timber.e("Parsing OpenRosa reply -- %s", error); + formList.clear(); + formList.put( + DL_ERROR_MSG, + new FormDetails(Collect.getInstance().getString( + R.string.parse_openrosa_formlist_failed, error))); + return formList; + } + boolean isNewerFormVersionAvailable = false; + boolean areNewerMediaFilesAvailable = false; + ManifestFile manifestFile = null; + if (isThisFormAlreadyDownloaded(formId)) { + isNewerFormVersionAvailable = isNewerFormVersionAvailable(FormDownloader.getMd5Hash(hash)); + if ((!isNewerFormVersionAvailable || alwaysCheckMediaFiles) && manifestUrl != null) { + manifestFile = getManifestFile(manifestUrl); + if (manifestFile != null) { + List newMediaFiles = manifestFile.getMediaFiles(); + if (newMediaFiles != null && newMediaFiles.size() > 0) { + areNewerMediaFilesAvailable = areNewerMediaFilesAvailable(formId, version, newMediaFiles); + } + } + } + } + formList.put(formId, new FormDetails(formName, downloadUrl, manifestUrl, formId, + (version != null) ? version : majorMinorVersion, hash, + manifestFile != null ? manifestFile.getHash() : null, + isNewerFormVersionAvailable, areNewerMediaFilesAvailable)); + } + } else { + // Aggregate 0.9.x mode... + // populate HashMap with form names and urls + Element formsElement = result.doc.getRootElement(); + int formsCount = formsElement.getChildCount(); + String formId = null; + for (int i = 0; i < formsCount; ++i) { + if (formsElement.getType(i) != Element.ELEMENT) { + // whitespace + continue; + } + Element child = formsElement.getElement(i); + String tag = child.getName(); + if (tag.equals("formID")) { + formId = XFormParser.getXMLText(child, true); + if (formId != null && formId.length() == 0) { + formId = null; + } + } + if (tag.equalsIgnoreCase("form")) { + String formName = XFormParser.getXMLText(child, true); + if (formName != null && formName.length() == 0) { + formName = null; + } + String downloadUrl = child.getAttributeValue(null, "url"); + downloadUrl = downloadUrl.trim(); + if (downloadUrl != null && downloadUrl.length() == 0) { + downloadUrl = null; + } + if (downloadUrl == null || formName == null) { + String error = + "Forms list entry " + Integer.toString(i) + + " is missing form name or url attribute"; + Timber.e("Parsing OpenRosa reply -- %s", error); + formList.clear(); + formList.put( + DL_ERROR_MSG, + new FormDetails(Collect.getInstance().getString( + R.string.parse_legacy_formlist_failed, error))); + return formList; + } + formList.put(formName, + new FormDetails(formName, downloadUrl, null, formId, null, null, null, false, false)); + + formId = null; + } + } + } + return formList; + } + + private static boolean isThisFormAlreadyDownloaded(String formId) { + return new FormsDao().getFormsCursorForFormId(formId).getCount() > 0; + } + + private static ManifestFile getManifestFile(String manifestUrl) { + if (manifestUrl == null) { + return null; + } + + // get shared HttpContext so that authentication and cookies are retained. + HttpContext localContext = Collect.getInstance().getHttpContext(); + + HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); + + DocumentFetchResult result = + WebUtils.getXmlDocument(manifestUrl, localContext, httpclient); + + if (result.errorMessage != null) { + return null; + } + + String errMessage = Collect.getInstance().getString(R.string.access_error, manifestUrl); + + if (!result.isOpenRosaResponse) { + errMessage += Collect.getInstance().getString(R.string.manifest_server_error); + Timber.e(errMessage); + return null; + } + + // Attempt OpenRosa 1.0 parsing + Element manifestElement = result.doc.getRootElement(); + if (!manifestElement.getName().equals("manifest")) { + errMessage += + Collect.getInstance().getString(R.string.root_element_error, + manifestElement.getName()); + Timber.e(errMessage); + return null; + } + String namespace = manifestElement.getNamespace(); + if (!FormDownloader.isXformsManifestNamespacedElement(manifestElement)) { + errMessage += Collect.getInstance().getString(R.string.root_namespace_error, namespace); + Timber.e(errMessage); + return null; + } + int elements = manifestElement.getChildCount(); + List files = new ArrayList<>(); + for (int i = 0; i < elements; ++i) { + if (manifestElement.getType(i) != Element.ELEMENT) { + // e.g., whitespace (text) + continue; + } + Element mediaFileElement = manifestElement.getElement(i); + if (!FormDownloader.isXformsManifestNamespacedElement(mediaFileElement)) { + // someone else's extension? + continue; + } + String name = mediaFileElement.getName(); + if (name.equalsIgnoreCase("mediaFile")) { + String filename = null; + String hash = null; + String downloadUrl = null; + // don't process descriptionUrl + int childCount = mediaFileElement.getChildCount(); + for (int j = 0; j < childCount; ++j) { + if (mediaFileElement.getType(j) != Element.ELEMENT) { + // e.g., whitespace (text) + continue; + } + Element child = mediaFileElement.getElement(j); + if (!FormDownloader.isXformsManifestNamespacedElement(child)) { + // someone else's extension? + continue; + } + String tag = child.getName(); + switch (tag) { + case "filename": + filename = XFormParser.getXMLText(child, true); + if (filename != null && filename.length() == 0) { + filename = null; + } + break; + case "hash": + hash = XFormParser.getXMLText(child, true); + if (hash != null && hash.length() == 0) { + hash = null; + } + break; + case "downloadUrl": + downloadUrl = XFormParser.getXMLText(child, true); + if (downloadUrl != null && downloadUrl.length() == 0) { + downloadUrl = null; + } + break; + } + } + if (filename == null || downloadUrl == null || hash == null) { + errMessage += + Collect.getInstance().getString(R.string.manifest_tag_error, + Integer.toString(i)); + Timber.e(errMessage); + return null; + } + files.add(new MediaFile(filename, hash, downloadUrl)); + } + } + + return new ManifestFile(result.getHash(), files); + } + + private static boolean isNewerFormVersionAvailable(String md5Hash) { + return md5Hash != null && new FormsDao().getFormsCursorForMd5Hash(md5Hash).getCount() == 0; + } + + private static boolean areNewerMediaFilesAvailable(String formId, String formVersion, List newMediaFiles) { + String mediaDirPath = new FormsDao().getFormMediaPath(formId, formVersion); + if (mediaDirPath != null) { + File[] localMediaFiles = new File(mediaDirPath).listFiles(); + if (localMediaFiles != null) { + for (MediaFile newMediaFile : newMediaFiles) { + if (!isMediaFileAlreadyDownloaded(localMediaFiles, newMediaFile)) { + return true; + } + } + } else if (!newMediaFiles.isEmpty()) { + return true; + } + } + + return false; + } + + private static boolean isMediaFileAlreadyDownloaded(File[] localMediaFiles, MediaFile newMediaFile) { + // TODO Zip files are ignored we should find a way to take them into account too + if (newMediaFile.getFilename().endsWith(".zip")) { + return true; + } + + String mediaFileHash = newMediaFile.getHash(); + mediaFileHash = mediaFileHash.substring(4, mediaFileHash.length()); + for (File localMediaFile : localMediaFiles) { + if (mediaFileHash.equals(FileUtils.getMd5Hash(localMediaFile))) { + return true; + } + } + return false; + } +} diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java b/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java index b91b64ab8a1..0475484bfc5 100644 --- a/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java @@ -136,7 +136,7 @@ public static String getMd5Hash(File file) { return getMd5Hash(is); } - private static String getMd5Hash(InputStream is) { + public static String getMd5Hash(InputStream is) { try { MessageDigest md = MessageDigest.getInstance("MD5"); final byte[] buffer = new byte[bufSize]; diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/FormDownloader.java b/collect_app/src/main/java/org/odk/collect/android/utilities/FormDownloader.java new file mode 100644 index 00000000000..e5e30cb14c8 --- /dev/null +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/FormDownloader.java @@ -0,0 +1,726 @@ +/* + * Copyright 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.odk.collect.android.utilities; + +import android.content.ContentValues; +import android.database.Cursor; +import android.net.Uri; + +import org.javarosa.xform.parse.XFormParser; +import org.kxml2.kdom.Element; +import org.odk.collect.android.R; +import org.odk.collect.android.application.Collect; +import org.odk.collect.android.dao.FormsDao; +import org.odk.collect.android.listeners.FormDownloaderListener; +import org.odk.collect.android.logic.FormDetails; +import org.odk.collect.android.logic.MediaFile; +import org.odk.collect.android.provider.FormsProviderAPI; +import org.opendatakit.httpclientandroidlib.Header; +import org.opendatakit.httpclientandroidlib.HttpEntity; +import org.opendatakit.httpclientandroidlib.HttpResponse; +import org.opendatakit.httpclientandroidlib.HttpStatus; +import org.opendatakit.httpclientandroidlib.client.HttpClient; +import org.opendatakit.httpclientandroidlib.client.methods.HttpGet; +import org.opendatakit.httpclientandroidlib.protocol.HttpContext; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; + +import timber.log.Timber; + +public class FormDownloader { + + private static final String MD5_COLON_PREFIX = "md5:"; + private static final String TEMP_DOWNLOAD_EXTENSION = ".tempDownload"; + + private FormDownloaderListener stateListener; + + private FormsDao formsDao; + + public void setDownloaderListener(FormDownloaderListener sl) { + synchronized (this) { + stateListener = sl; + } + } + + private static final String NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_MANIFEST = + "http://openrosa.org/xforms/xformsManifest"; + + static boolean isXformsManifestNamespacedElement(Element e) { + return e.getNamespace().equalsIgnoreCase(NAMESPACE_OPENROSA_ORG_XFORMS_XFORMS_MANIFEST); + } + + private class TaskCancelledException extends Exception { + private final File file; + + TaskCancelledException(File file) { + super("Task was cancelled during processing of " + file); + this.file = file; + } + + TaskCancelledException() { + super("Task was cancelled"); + this.file = null; + } + } + + public HashMap downloadForms(List toDownload) { + formsDao = new FormsDao(); + int total = toDownload.size(); + int count = 1; + Collect.getInstance().getActivityLogger().logAction(this, "downloadForms", + String.valueOf(total)); + + final HashMap result = new HashMap<>(); + + for (FormDetails fd : toDownload) { + try { + String message = processOneForm(total, count++, fd); + result.put(fd, message.isEmpty() ? + Collect.getInstance().getString(R.string.success) : message); + } catch (TaskCancelledException cd) { + break; + } + } + + return result; + } + + /** + * Processes one form download. + * + * @param total the total number of forms being downloaded by this task + * @param count the number of this form + * @param fd the FormDetails + * @return an empty string for success, or a nonblank string with one or more error messages + * @throws TaskCancelledException to signal that form downloading is to be canceled + */ + private String processOneForm(int total, int count, FormDetails fd) throws TaskCancelledException { + if (stateListener != null) { + stateListener.progressUpdate(fd.getFormName(), String.valueOf(count), String.valueOf(total)); + } + String message = ""; + if (stateListener != null && stateListener.isTaskCanceled()) { + throw new TaskCancelledException(); + } + + String tempMediaPath = null; + final String finalMediaPath; + FileResult fileResult = null; + try { + // get the xml file + // if we've downloaded a duplicate, this gives us the file + fileResult = downloadXform(fd.getFormName(), fd.getDownloadUrl()); + + if (fd.getManifestUrl() != null) { + // use a temporary media path until everything is ok. + tempMediaPath = new File(Collect.CACHE_PATH, + String.valueOf(System.currentTimeMillis())).getAbsolutePath(); + finalMediaPath = FileUtils.constructMediaPath( + fileResult.getFile().getAbsolutePath()); + String error = downloadManifestAndMediaFiles(tempMediaPath, finalMediaPath, fd, + count, total); + if (error != null) { + message += error; + } + } else { + Timber.i("No Manifest for: %s", fd.getFormName()); + } + } catch (TaskCancelledException e) { + Timber.i(e); + cleanUp(fileResult, e.file, tempMediaPath); + + // do not download additional forms. + throw e; + } catch (Exception e) { + message += getExceptionMessage(e); + } + + if (stateListener != null && stateListener.isTaskCanceled()) { + cleanUp(fileResult, null, tempMediaPath); + fileResult = null; + } + + Map parsedFields = null; + if (fileResult != null) { + try { + final long start = System.currentTimeMillis(); + Timber.w("Parsing document %s", fileResult.file.getAbsolutePath()); + parsedFields = FileUtils.parseXML(fileResult.file); + Timber.i("Parse finished in %.3f seconds.", + (System.currentTimeMillis() - start) / 1000F); + } catch (RuntimeException e) { + message += e.getMessage(); + } + } + + boolean installed = false; + + if ((stateListener == null || !stateListener.isTaskCanceled()) && message.isEmpty() && parsedFields != null) { + if (isSubmissionOk(parsedFields)) { + installEverything(tempMediaPath, fileResult, parsedFields); + installed = true; + } else { + message += Collect.getInstance().getString(R.string.xform_parse_error, + fileResult.file.getName(), "submission url"); + } + } + if (!installed) { + cleanUp(fileResult, null, tempMediaPath); + } + return message; + } + + private boolean isSubmissionOk(Map parsedFields) { + String submission = parsedFields.get(FileUtils.SUBMISSIONURI); + return submission == null || Validator.isUrlValid(submission); + } + + private void installEverything(String tempMediaPath, FileResult fileResult, Map parsedFields) { + UriResult uriResult = null; + try { + uriResult = findExistingOrCreateNewUri(fileResult.file, parsedFields); + Timber.w("Form uri = %s, isNew = %b", uriResult.getUri().toString(), uriResult.isNew()); + + // move the media files in the media folder + if (tempMediaPath != null) { + File formMediaPath = new File(uriResult.getMediaPath()); + FileUtils.moveMediaFiles(tempMediaPath, formMediaPath); + } + } catch (IOException e) { + Timber.e(e); + + if (uriResult != null && uriResult.isNew() && fileResult.isNew()) { + // this means we should delete the entire form together with the metadata + Uri uri = uriResult.getUri(); + Timber.w("The form is new. We should delete the entire form."); + int deletedCount = Collect.getInstance().getContentResolver().delete(uri, + null, null); + Timber.w("Deleted %d rows using uri %s", deletedCount, uri.toString()); + } + + cleanUp(fileResult, null, tempMediaPath); + } + } + + private void cleanUp(FileResult fileResult, File fileOnCancel, String tempMediaPath) { + if (fileResult == null) { + Timber.w("The user cancelled (or an exception happened) the download of a form at the " + + "very beginning."); + } else { + formsDao.deleteFormsFromMd5Hash(FileUtils.getMd5Hash(fileResult.file)); + FileUtils.deleteAndReport(fileResult.getFile()); + } + + FileUtils.deleteAndReport(fileOnCancel); + + if (tempMediaPath != null) { + FileUtils.purgeMediaPath(tempMediaPath); + } + } + + private String getExceptionMessage(Exception e) { + String msg = e.getMessage(); + if (msg == null) { + msg = e.toString(); + } + Timber.e(msg); + + if (e.getCause() != null) { + msg = e.getCause().getMessage(); + if (msg == null) { + msg = e.getCause().toString(); + } + } + return msg; + } + + /** + * Creates a new form in the database, if none exists with the same absolute path. Returns + * information with the URI, media path, and whether the form is new. + * + * @param formFile the form definition file + * @param formInfo certain fields extracted from the parsed XML form, such as title and form ID + * @return a {@link UriResult} object + */ + private UriResult findExistingOrCreateNewUri(File formFile, Map formInfo) { + Cursor cursor = null; + final Uri uri; + final String formFilePath = formFile.getAbsolutePath(); + String mediaPath = FileUtils.constructMediaPath(formFilePath); + final boolean isNew; + + FileUtils.checkMediaPath(new File(mediaPath)); + + try { + cursor = formsDao.getFormsCursorForFormFilePath(formFile.getAbsolutePath()); + isNew = cursor.getCount() <= 0; + + if (isNew) { + uri = saveNewForm(formInfo, formFile, mediaPath); + } else { + cursor.moveToFirst(); + uri = Uri.withAppendedPath(FormsProviderAPI.FormsColumns.CONTENT_URI, + cursor.getString(cursor.getColumnIndex(FormsProviderAPI.FormsColumns._ID))); + mediaPath = cursor.getString(cursor.getColumnIndex(FormsProviderAPI.FormsColumns.FORM_MEDIA_PATH)); + Collect.getInstance().getActivityLogger().logAction(this, "refresh", + formFilePath); + } + } finally { + if (cursor != null) { + cursor.close(); + } + } + + return new UriResult(uri, mediaPath, isNew); + } + + private Uri saveNewForm(Map formInfo, File formFile, String mediaPath) { + final ContentValues v = new ContentValues(); + v.put(FormsProviderAPI.FormsColumns.FORM_FILE_PATH, formFile.getAbsolutePath()); + v.put(FormsProviderAPI.FormsColumns.FORM_MEDIA_PATH, mediaPath); + v.put(FormsProviderAPI.FormsColumns.DISPLAY_NAME, formInfo.get(FileUtils.TITLE)); + v.put(FormsProviderAPI.FormsColumns.JR_VERSION, formInfo.get(FileUtils.VERSION)); + v.put(FormsProviderAPI.FormsColumns.JR_FORM_ID, formInfo.get(FileUtils.FORMID)); + v.put(FormsProviderAPI.FormsColumns.SUBMISSION_URI, formInfo.get(FileUtils.SUBMISSIONURI)); + v.put(FormsProviderAPI.FormsColumns.BASE64_RSA_PUBLIC_KEY, formInfo.get(FileUtils.BASE64_RSA_PUBLIC_KEY)); + v.put(FormsProviderAPI.FormsColumns.AUTO_DELETE, formInfo.get(FileUtils.AUTO_DELETE)); + v.put(FormsProviderAPI.FormsColumns.AUTO_SEND, formInfo.get(FileUtils.AUTO_SEND)); + Uri uri = formsDao.saveForm(v); + Collect.getInstance().getActivityLogger().logAction(this, "insert", + formFile.getAbsolutePath()); + return uri; + } + + /** + * Takes the formName and the URL and attempts to download the specified file. Returns a file + * object representing the downloaded file. + */ + private FileResult downloadXform(String formName, String url) + throws IOException, TaskCancelledException, Exception { + // clean up friendly form name... + String rootName = formName.replaceAll("[^\\p{L}\\p{Digit}]", " "); + rootName = rootName.replaceAll("\\p{javaWhitespace}+", " "); + rootName = rootName.trim(); + + // proposed name of xml file... + String path = Collect.FORMS_PATH + File.separator + rootName + ".xml"; + int i = 2; + File f = new File(path); + while (f.exists()) { + path = Collect.FORMS_PATH + File.separator + rootName + "_" + i + ".xml"; + f = new File(path); + i++; + } + + downloadFile(f, url); + + boolean isNew = true; + + // we've downloaded the file, and we may have renamed it + // make sure it's not the same as a file we already have + Cursor c = null; + try { + c = formsDao.getFormsCursorForMd5Hash(FileUtils.getMd5Hash(f)); + if (c.getCount() > 0) { + // Should be at most, 1 + c.moveToFirst(); + + isNew = false; + + // delete the file we just downloaded, because it's a duplicate + Timber.w("A duplicate file has been found, we need to remove the downloaded file " + + "and return the other one."); + FileUtils.deleteAndReport(f); + + // set the file returned to the file we already had + String existingPath = c.getString(c.getColumnIndex(FormsProviderAPI.FormsColumns.FORM_FILE_PATH)); + f = new File(existingPath); + Timber.w("Will use %s", existingPath); + } + } finally { + if (c != null) { + c.close(); + } + } + + return new FileResult(f, isNew); + } + + /** + * Common routine to download a document from the downloadUrl and save the contents in the file + * 'file'. Shared by media file download and form file download. + *

+ * SurveyCTO: The file is saved into a temp folder and is moved to the final place if everything + * is okay, so that garbage is not left over on cancel. + * + * @param file the final file + * @param downloadUrl the url to get the contents from. + */ + private void downloadFile(File file, String downloadUrl) + throws IOException, TaskCancelledException, URISyntaxException, Exception { + File tempFile = File.createTempFile(file.getName(), TEMP_DOWNLOAD_EXTENSION, + new File(Collect.CACHE_PATH)); + + URI uri; + try { + // assume the downloadUrl is escaped properly + URL url = new URL(downloadUrl); + uri = url.toURI(); + } catch (MalformedURLException | URISyntaxException e) { + Timber.e(e, "Unable to get a URI for download URL : %s due to %s : ", downloadUrl, e.getMessage()); + throw e; + } + + // WiFi network connections can be renegotiated during a large form download sequence. + // This will cause intermittent download failures. Silently retry once after each + // failure. Only if there are two consecutive failures do we abort. + boolean success = false; + int attemptCount = 0; + final int MAX_ATTEMPT_COUNT = 2; + while (!success && ++attemptCount <= MAX_ATTEMPT_COUNT) { + if (stateListener != null && stateListener.isTaskCanceled()) { + throw new TaskCancelledException(tempFile); + } + Timber.i("Started downloading to %s from %s", tempFile.getAbsolutePath(), downloadUrl); + + // get shared HttpContext so that authentication and cookies are retained. + HttpContext localContext = Collect.getInstance().getHttpContext(); + + HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); + + // set up request... + HttpGet req = WebUtils.createOpenRosaHttpGet(uri); + req.addHeader(WebUtils.ACCEPT_ENCODING_HEADER, WebUtils.GZIP_CONTENT_ENCODING); + + HttpResponse response; + try { + response = httpclient.execute(req, localContext); + int statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode != HttpStatus.SC_OK) { + WebUtils.discardEntityBytes(response); + if (statusCode == HttpStatus.SC_UNAUTHORIZED) { + // clear the cookies -- should not be necessary? + Collect.getInstance().getCookieStore().clear(); + } + String errMsg = + Collect.getInstance().getString(R.string.file_fetch_failed, downloadUrl, + response.getStatusLine().getReasonPhrase(), String.valueOf(statusCode)); + Timber.e(errMsg); + throw new Exception(errMsg); + } + + // write connection to file + InputStream is = null; + OutputStream os = null; + try { + HttpEntity entity = response.getEntity(); + is = entity.getContent(); + Header contentEncoding = entity.getContentEncoding(); + if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase( + WebUtils.GZIP_CONTENT_ENCODING)) { + is = new GZIPInputStream(is); + } + os = new FileOutputStream(tempFile); + byte[] buf = new byte[4096]; + int len; + while ((len = is.read(buf)) > 0 && (stateListener == null || !stateListener.isTaskCanceled())) { + os.write(buf, 0, len); + } + os.flush(); + success = true; + } finally { + if (os != null) { + try { + os.close(); + } catch (Exception e) { + Timber.e(e); + } + } + if (is != null) { + try { + // ensure stream is consumed... + final long count = 1024L; + while (is.skip(count) == count) { + // skipping to the end of the http entity + } + } catch (Exception e) { + // no-op + } + try { + is.close(); + } catch (Exception e) { + Timber.e(e); + } + } + } + } catch (Exception e) { + Timber.e(e.toString()); + // silently retry unless this is the last attempt, + // in which case we rethrow the exception. + + FileUtils.deleteAndReport(tempFile); + + if (attemptCount == MAX_ATTEMPT_COUNT) { + throw e; + } + } + + if (stateListener != null && stateListener.isTaskCanceled()) { + FileUtils.deleteAndReport(tempFile); + throw new TaskCancelledException(tempFile); + } + } + + Timber.d("Completed downloading of %s. It will be moved to the proper path...", + tempFile.getAbsolutePath()); + + FileUtils.deleteAndReport(file); + + String errorMessage = FileUtils.copyFile(tempFile, file); + + if (file.exists()) { + Timber.w("Copied %s over %s", tempFile.getAbsolutePath(), file.getAbsolutePath()); + FileUtils.deleteAndReport(tempFile); + } else { + String msg = Collect.getInstance().getString(R.string.fs_file_copy_error, + tempFile.getAbsolutePath(), file.getAbsolutePath(), errorMessage); + Timber.w(msg); + throw new RuntimeException(msg); + } + } + + private static class UriResult { + + private final Uri uri; + private final String mediaPath; + private final boolean isNew; + + private UriResult(Uri uri, String mediaPath, boolean isNew) { + this.uri = uri; + this.mediaPath = mediaPath; + this.isNew = isNew; + } + + private Uri getUri() { + return uri; + } + + private String getMediaPath() { + return mediaPath; + } + + private boolean isNew() { + return isNew; + } + } + + private static class FileResult { + + private final File file; + private final boolean isNew; + + private FileResult(File file, boolean isNew) { + this.file = file; + this.isNew = isNew; + } + + private File getFile() { + return file; + } + + private boolean isNew() { + return isNew; + } + } + + private String downloadManifestAndMediaFiles(String tempMediaPath, String finalMediaPath, + FormDetails fd, int count, + int total) throws Exception { + if (fd.getManifestUrl() == null) { + return null; + } + + if (stateListener != null) { + stateListener.progressUpdate(Collect.getInstance().getString(R.string.fetching_manifest, fd.getFormName()), + String.valueOf(count), String.valueOf(total)); + } + + List files = new ArrayList(); + // get shared HttpContext so that authentication and cookies are retained. + HttpContext localContext = Collect.getInstance().getHttpContext(); + + HttpClient httpclient = WebUtils.createHttpClient(WebUtils.CONNECTION_TIMEOUT); + + DocumentFetchResult result = + WebUtils.getXmlDocument(fd.getManifestUrl(), localContext, httpclient); + + if (result.errorMessage != null) { + return result.errorMessage; + } + + String errMessage = Collect.getInstance().getString(R.string.access_error, fd.getManifestUrl()); + + if (!result.isOpenRosaResponse) { + errMessage += Collect.getInstance().getString(R.string.manifest_server_error); + Timber.e(errMessage); + return errMessage; + } + + // Attempt OpenRosa 1.0 parsing + Element manifestElement = result.doc.getRootElement(); + if (!manifestElement.getName().equals("manifest")) { + errMessage += + Collect.getInstance().getString(R.string.root_element_error, + manifestElement.getName()); + Timber.e(errMessage); + return errMessage; + } + String namespace = manifestElement.getNamespace(); + if (!isXformsManifestNamespacedElement(manifestElement)) { + errMessage += Collect.getInstance().getString(R.string.root_namespace_error, namespace); + Timber.e(errMessage); + return errMessage; + } + int elements = manifestElement.getChildCount(); + for (int i = 0; i < elements; ++i) { + if (manifestElement.getType(i) != Element.ELEMENT) { + // e.g., whitespace (text) + continue; + } + Element mediaFileElement = manifestElement.getElement(i); + if (!isXformsManifestNamespacedElement(mediaFileElement)) { + // someone else's extension? + continue; + } + String name = mediaFileElement.getName(); + if (name.equalsIgnoreCase("mediaFile")) { + String filename = null; + String hash = null; + String downloadUrl = null; + // don't process descriptionUrl + int childCount = mediaFileElement.getChildCount(); + for (int j = 0; j < childCount; ++j) { + if (mediaFileElement.getType(j) != Element.ELEMENT) { + // e.g., whitespace (text) + continue; + } + Element child = mediaFileElement.getElement(j); + if (!isXformsManifestNamespacedElement(child)) { + // someone else's extension? + continue; + } + String tag = child.getName(); + switch (tag) { + case "filename": + filename = XFormParser.getXMLText(child, true); + if (filename != null && filename.length() == 0) { + filename = null; + } + break; + case "hash": + hash = XFormParser.getXMLText(child, true); + if (hash != null && hash.length() == 0) { + hash = null; + } + break; + case "downloadUrl": + downloadUrl = XFormParser.getXMLText(child, true); + if (downloadUrl != null && downloadUrl.length() == 0) { + downloadUrl = null; + } + break; + } + } + if (filename == null || downloadUrl == null || hash == null) { + errMessage += + Collect.getInstance().getString(R.string.manifest_tag_error, + Integer.toString(i)); + Timber.e(errMessage); + return errMessage; + } + files.add(new MediaFile(filename, hash, downloadUrl)); + } + } + + // OK we now have the full set of files to download... + Timber.i("Downloading %d media files.", files.size()); + int mediaCount = 0; + if (files.size() > 0) { + File tempMediaDir = new File(tempMediaPath); + File finalMediaDir = new File(finalMediaPath); + + FileUtils.checkMediaPath(tempMediaDir); + FileUtils.checkMediaPath(finalMediaDir); + + for (MediaFile toDownload : files) { + ++mediaCount; + if (stateListener != null) { + stateListener.progressUpdate( + Collect.getInstance().getString(R.string.form_download_progress, + fd.getFormName(), + String.valueOf(mediaCount), String.valueOf(files.size())), + String.valueOf(count), String.valueOf(total)); + } + + //try { + File finalMediaFile = new File(finalMediaDir, toDownload.getFilename()); + File tempMediaFile = new File(tempMediaDir, toDownload.getFilename()); + + if (!finalMediaFile.exists()) { + downloadFile(tempMediaFile, toDownload.getDownloadUrl()); + } else { + String currentFileHash = FileUtils.getMd5Hash(finalMediaFile); + String downloadFileHash = getMd5Hash(toDownload.getHash()); + + if (currentFileHash != null && downloadFileHash != null && !currentFileHash.contentEquals(downloadFileHash)) { + // if the hashes match, it's the same file + // otherwise delete our current one and replace it with the new one + FileUtils.deleteAndReport(finalMediaFile); + downloadFile(tempMediaFile, toDownload.getDownloadUrl()); + } else { + // exists, and the hash is the same + // no need to download it again + Timber.i("Skipping media file fetch -- file hashes identical: %s", + finalMediaFile.getAbsolutePath()); + } + } + // } catch (Exception e) { + // return e.getLocalizedMessage(); + //} + } + } + return null; + } + + public static String getMd5Hash(String hash) { + return hash == null || hash.isEmpty() ? null : hash.substring(MD5_COLON_PREFIX.length()); + } +} diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/ServerPollingJobCreator.java b/collect_app/src/main/java/org/odk/collect/android/utilities/ServerPollingJobCreator.java new file mode 100644 index 00000000000..51794272772 --- /dev/null +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/ServerPollingJobCreator.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.odk.collect.android.utilities; + +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + +import com.evernote.android.job.Job; +import com.evernote.android.job.JobCreator; + +import org.odk.collect.android.tasks.ServerPollingJob; + +public class ServerPollingJobCreator implements JobCreator { + + @Override + @Nullable + public Job create(@NonNull String tag) { + return tag.equals(ServerPollingJob.TAG) ? new ServerPollingJob() : null; + } +} \ No newline at end of file diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/SharedPreferencesUtils.java b/collect_app/src/main/java/org/odk/collect/android/utilities/SharedPreferencesUtils.java index c887d5ca314..139ab98b914 100644 --- a/collect_app/src/main/java/org/odk/collect/android/utilities/SharedPreferencesUtils.java +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/SharedPreferencesUtils.java @@ -36,7 +36,6 @@ import static org.odk.collect.android.preferences.PreferenceKeys.GENERAL_KEYS; import static org.odk.collect.android.preferences.PreferenceKeys.KEY_PASSWORD; - public final class SharedPreferencesUtils { private SharedPreferencesUtils() { @@ -102,7 +101,6 @@ public static void savePreferencesFromString(String content, ActionListener list JSONObject adminPrefsJson = settingsJson.getJSONObject("admin"); for (String key : getAllGeneralKeys()) { - if (generalPrefsJson.has(key)) { Object value = generalPrefsJson.get(key); GeneralSharedPreferences.getInstance().save(key, value); diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/WebUtils.java b/collect_app/src/main/java/org/odk/collect/android/utilities/WebUtils.java index dde4d08ceeb..248eff40a4c 100644 --- a/collect_app/src/main/java/org/odk/collect/android/utilities/WebUtils.java +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/WebUtils.java @@ -17,6 +17,7 @@ import android.net.Uri; import android.text.format.DateFormat; +import org.apache.commons.io.IOUtils; import org.kxml2.io.KXmlParser; import org.kxml2.kdom.Document; import org.odk.collect.android.BuildConfig; @@ -47,6 +48,7 @@ import org.opendatakit.httpclientandroidlib.protocol.HttpContext; import org.xmlpull.v1.XmlPullParser; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -336,11 +338,14 @@ public static DocumentFetchResult getXmlDocument(String urlString, } // parse response Document doc = null; + String hash; try { InputStream is = null; InputStreamReader isr = null; try { - is = entity.getContent(); + byte[] bytes = IOUtils.toByteArray(entity.getContent()); + is = new ByteArrayInputStream(bytes); + hash = FileUtils.getMd5Hash(new ByteArrayInputStream(bytes)); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase( WebUtils.GZIP_CONTENT_ENCODING)) { @@ -413,7 +418,7 @@ public static DocumentFetchResult getXmlDocument(String urlString, Timber.w("%s unrecognized version(s): %s", WebUtils.OPEN_ROSA_VERSION_HEADER, b.toString()); } } - return new DocumentFetchResult(doc, isOR); + return new DocumentFetchResult(doc, isOR, hash); } catch (Exception e) { String cause; Throwable c = e; diff --git a/collect_app/src/main/res/values/arrays.xml b/collect_app/src/main/res/values/arrays.xml index 0deeefbe75c..1268b02d96f 100644 --- a/collect_app/src/main/res/values/arrays.xml +++ b/collect_app/src/main/res/values/arrays.xml @@ -194,4 +194,18 @@ the specific language governing permissions and limitations under the License. - @string/islamic_month_11 @string/islamic_month_12 + + @string/never_value + @string/every_fifteen_minutes_value + @string/every_one_hour_value + @string/every_six_hours_value + @string/every_24_hours_value + + + @string/never + @string/every_fifteen_minutes + @string/every_one_hour + @string/every_six_hours + @string/every_24_hours + diff --git a/collect_app/src/main/res/values/strings.xml b/collect_app/src/main/res/values/strings.xml index 606c1a3792b..3b3a2394f71 100644 --- a/collect_app/src/main/res/values/strings.xml +++ b/collect_app/src/main/res/values/strings.xml @@ -586,6 +586,19 @@ Tap the screen to start recording Recording started. Tap again to stop No columns found in the form to upload. + Form update + Periodic form updates check + Never + Every fifteen minutes + Every one hour + Every six hours + Every 24 hours + Form updates available + New form versions downloaded + Automatic download + Automatically download updated versions of forms + Forms downloaded: + Forms sent: Show guidance for questions Theme Light theme diff --git a/collect_app/src/main/res/values/untranslated.xml b/collect_app/src/main/res/values/untranslated.xml index 931b445b5b4..12a5354119e 100644 --- a/collect_app/src/main/res/values/untranslated.xml +++ b/collect_app/src/main/res/values/untranslated.xml @@ -16,6 +16,11 @@ the specific language governing permissions and limitations under the License. - /formList /submission http://opendatakit.org + never + every_fifteen_minutes + every_one_hour + every_six_hours + every_24_hours light_theme dark_theme diff --git a/collect_app/src/main/res/xml/form_management_preferences.xml b/collect_app/src/main/res/xml/form_management_preferences.xml index 2df7cc822e8..d46216e2142 100644 --- a/collect_app/src/main/res/xml/form_management_preferences.xml +++ b/collect_app/src/main/res/xml/form_management_preferences.xml @@ -1,5 +1,19 @@ + + + + diff --git a/collect_app/src/main/res/xml/user_settings_access_preferences.xml b/collect_app/src/main/res/xml/user_settings_access_preferences.xml index a18ccfddf0a..1ea011d6758 100644 --- a/collect_app/src/main/res/xml/user_settings_access_preferences.xml +++ b/collect_app/src/main/res/xml/user_settings_access_preferences.xml @@ -28,6 +28,12 @@ + +