<NME> GpxExport.java
<BEF> package cgeo.geocaching.export;

import cgeo.geocaching.CgeoApplication;
import cgeo.geocaching.R;
import cgeo.geocaching.activity.ActivityMixin;
import cgeo.geocaching.models.Geocache;
import cgeo.geocaching.settings.Settings;
import cgeo.geocaching.storage.ContentStorage;
import cgeo.geocaching.storage.PersistableFolder;
import cgeo.geocaching.ui.dialog.Dialogs;
import cgeo.geocaching.utils.AsyncTaskWithProgress;
import cgeo.geocaching.utils.FileNameCreator;
import cgeo.geocaching.utils.Log;
import cgeo.geocaching.utils.ShareUtils;
import cgeo.geocaching.utils.UriUtils;

import android.app.Activity;
import android.app.Dialog;
import android.net.Uri;
import android.view.View;
import android.widget.CheckBox;
import android.widget.TextView;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AlertDialog;

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.commons.io.IOUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;

class GpxExport extends AbstractExport {
    private static final SimpleDateFormat dateFormatZ = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
    public void export(@NonNull final List<Geocache> caches, @Nullable final Activity activity, @Nullable final String title) {
    public static final String PREFIX_GPX = "http://www.topografix.com/GPX/1/0";
    public static final String PREFIX_GROUNDSPEAK = "http://www.groundspeak.com/cache/1/0";

    protected GpxExport() {
        super(getString(R.string.export_gpx));
    }
            // No activity given, so no user interaction possible.
            // Start export with default parameters.
            new ExportTask(null).execute(geocodes);

        } else {
            // Show configuration dialog
            getExportDialog(geocodes, activity).show();
        }
    }

    private void calculateFileName(final String[] geocodes) {
        if (geocodes.length == 1) {
            // geocode as file name
            fileName = geocodes[0] + (StringUtils.isNotBlank(title) ? " " + title : "") + ".gpx";
        } else {
            fileName = FileNameCreator.GPX_EXPORT.createName();
            if (StringUtils.isNotBlank(title)) {
                final int pos = fileName.lastIndexOf(".");
                if (pos >= 0) {
                    final String first = fileName.substring(0, pos);
                    final String last = fileName.substring(pos);
                    fileName = first + " " + title + last;
                } else {
                    fileName += " " + title;
                }
            }
        }
    }

    private Dialog getExportDialog(final String[] geocodes, final Activity activity) {
        final AlertDialog.Builder builder = Dialogs.newBuilder(activity);
        builder.setTitle(activity.getString(R.string.export_confirm_title, activity.getString(R.string.export_gpx)));

        final View layout = View.inflate(activity, R.layout.gpx_export_dialog, null);
        builder.setView(layout);

        final TextView text = layout.findViewById(R.id.info);
        text.setText(activity.getString(R.string.export_confirm_message, PersistableFolder.GPX.toUserDisplayableValue(), fileName));

        final CheckBox includeFoundStatus = layout.findViewById(R.id.include_found_status);
        includeFoundStatus.setChecked(Settings.getIncludeFoundStatus());

        builder.setPositiveButton(R.string.export, (dialog, which) -> {
            Settings.setIncludeFoundStatus(includeFoundStatus.isChecked());
            dialog.dismiss();
    }

    private class ExportTask extends AsyncTaskWithProgress<Void, File> {
        private final List<Geocache> caches;
        private final Activity activity;

        /**
         * Instantiates and configures the task for exporting field notes.

    protected class ExportTask extends AsyncTaskWithProgress<String, Uri> {

        /**
         * Instantiates and configures the task for exporting field notes.
         */
        public ExportTask(final List<Geocache> caches, final Activity activity) {
            super(activity, caches.size(), getProgressTitle(), cgeoapplication.getInstance().getResources().getQuantityString(R.plurals.cache_counts, caches.size(), caches.size()));
            this.caches = caches;
            this.activity = activity;
        }

            setMessage(CgeoApplication.getInstance().getResources().getQuantityString(R.plurals.cache_counts, allGeocodes.size(), allGeocodes.size()));

            final Uri uri = ContentStorage.get().create(PersistableFolder.GPX, fileName);
            if (uri == null) {
                return null;
            }

            BufferedWriter writer = null;
            try (OutputStream os = ContentStorage.get().openForWrite(uri)) {
                if (os == null) {
                    return null;
                }

                writer = new BufferedWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8));
                new GpxSerializer().writeGPX(allGeocodes, writer, ExportTask.this::publishProgress);
            } catch (final IOException e) {
                Log.e("GpxExport.ExportTask export", e);
                // delete partial GPX file on error
                ContentStorage.get().delete(uri);

                return null;
            } finally {
                IOUtils.closeQuietly(writer);
            }

            return uri;
                        PREFIX_GPX + " http://www.topografix.com/GPX/1/0/gpx.xsd " +
                                PREFIX_GROUNDSPEAK + " http://www.groundspeak.com/cache/1/0/1/cache.xsd");

                for (int i = 0; i < caches.size(); i++) {
                    final Geocache cache = cgData.loadCache(caches.get(i).getGeocode(), LoadFlags.LOAD_ALL_DB_ONLY);

                    gpx.startTag(PREFIX_GPX, "wpt");
                    gpx.attribute("", "lat", Double.toString(cache.getCoords().getLatitude()));
                    gpx.attribute("", "lon", Double.toString(cache.getCoords().getLongitude()));

                    final Date hiddenDate = cache.getHiddenDate();
                    if (hiddenDate != null) {
                        XmlUtils.simpleText(gpx, PREFIX_GPX, "time", dateFormatZ.format(hiddenDate));
                    }

                    XmlUtils.multipleTexts(gpx, PREFIX_GPX,
                            "name", cache.getGeocode(),
                            "desc", cache.getName(),
                            "url", cache.getUrl(),
                            "urlname", cache.getName(),
                            "sym", cache.isFound() ? "Geocache Found" : "Geocache",
                            "type", "Geocache|" + cache.getType().pattern);

                    gpx.startTag(PREFIX_GROUNDSPEAK, "cache");
                    gpx.attribute("", "id", cache.getCacheId());
                    gpx.attribute("", "available", !cache.isDisabled() ? "True" : "False");
                    gpx.attribute("", "archives", cache.isArchived() ? "True" : "False");

                    XmlUtils.multipleTexts(gpx, PREFIX_GROUNDSPEAK,
                            "name", cache.getName(),
                            "placed_by", cache.getOwnerDisplayName(),
                            "owner", cache.getOwnerUserId(),
                            "type", cache.getType().pattern,
                            "container", cache.getSize().id,
                            "difficulty", Float.toString(cache.getDifficulty()),
                            "terrain", Float.toString(cache.getTerrain()),
                            "country", cache.getLocation(),
                            "state", "",
                            "encoded_hints", cache.getHint());

                    writeAttributes(gpx, cache);

                    gpx.startTag(PREFIX_GROUNDSPEAK, "short_description");
                    gpx.attribute("", "html", BaseUtils.containsHtml(cache.getShortDescription()) ? "True" : "False");
                    gpx.text(cache.getShortDescription());
                    gpx.endTag(PREFIX_GROUNDSPEAK, "short_description");

                    gpx.startTag(PREFIX_GROUNDSPEAK, "long_description");
                    gpx.attribute("", "html", BaseUtils.containsHtml(cache.getDescription()) ? "True" : "False");
                    gpx.text(cache.getDescription());
                    gpx.endTag(PREFIX_GROUNDSPEAK, "long_description");

                    writeLogs(gpx, cache);

                    gpx.endTag(PREFIX_GROUNDSPEAK, "cache");
                    gpx.endTag(PREFIX_GPX, "wpt");

                    writeWaypoints(gpx, cache);

                    publishProgress(i + 1);
                }

                gpx.endTag(PREFIX_GPX, "gpx");
                gpx.endDocument();
            } catch (final IOException e) {
                Log.e("GpxExport.ExportTask export", e);

                if (writer != null) {
            return exportFile;
        }

        private void writeWaypoints(final XmlSerializer gpx, final Geocache cache) throws IOException {
            List<Waypoint> waypoints = cache.getWaypoints();
            List<Waypoint> ownWaypoints = new ArrayList<Waypoint>(waypoints.size());

<MSG> refactoring: improve performance of GPX export

* load batches of caches instead of single caches
* do not load all caches to avoid huge memory usage
<DFF> @@ -38,8 +38,11 @@ import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 class GpxExport extends AbstractExport {
     private static final SimpleDateFormat dateFormatZ = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
@@ -47,6 +50,11 @@ class GpxExport extends AbstractExport {
     public static final String PREFIX_GPX = "http://www.topografix.com/GPX/1/0";
     public static final String PREFIX_GROUNDSPEAK = "http://www.groundspeak.com/cache/1/0";
 
+    /**
+     * During the export, only this number of geocaches is fully loaded into memory.
+     */
+    public static final int CACHES_PER_BATCH = 100;
+
     protected GpxExport() {
         super(getString(R.string.export_gpx));
     }
@@ -98,8 +106,9 @@ class GpxExport extends AbstractExport {
     }
 
     private class ExportTask extends AsyncTaskWithProgress<Void, File> {
-        private final List<Geocache> caches;
+        private final Set<String> allGeocodes = new HashSet<String>();
         private final Activity activity;
+        private int countExported = 0;
 
         /**
          * Instantiates and configures the task for exporting field notes.
@@ -111,7 +120,11 @@ class GpxExport extends AbstractExport {
          */
         public ExportTask(final List<Geocache> caches, final Activity activity) {
             super(activity, caches.size(), getProgressTitle(), cgeoapplication.getInstance().getResources().getQuantityString(R.plurals.cache_counts, caches.size(), caches.size()));
-            this.caches = caches;
+
+            // get rid of the (half loaded) caches, we will reload them as full caches during the export
+            for (Geocache geocache : caches) {
+                allGeocodes.add(geocache.getGeocode());
+            }
             this.activity = activity;
         }
 
@@ -144,68 +157,24 @@ class GpxExport extends AbstractExport {
                         PREFIX_GPX + " http://www.topografix.com/GPX/1/0/gpx.xsd " +
                                 PREFIX_GROUNDSPEAK + " http://www.groundspeak.com/cache/1/0/1/cache.xsd");
 
-                for (int i = 0; i < caches.size(); i++) {
-                    final Geocache cache = cgData.loadCache(caches.get(i).getGeocode(), LoadFlags.LOAD_ALL_DB_ONLY);
+                // Split the overall set of geocodes into small chunks. That is a compromise between memory efficiency (because
+                // we don't load all caches fully into memory) and speed (because we don't query each cache separately).
 
-                    gpx.startTag(PREFIX_GPX, "wpt");
-                    gpx.attribute("", "lat", Double.toString(cache.getCoords().getLatitude()));
-                    gpx.attribute("", "lon", Double.toString(cache.getCoords().getLongitude()));
-
-                    final Date hiddenDate = cache.getHiddenDate();
-                    if (hiddenDate != null) {
-                        XmlUtils.simpleText(gpx, PREFIX_GPX, "time", dateFormatZ.format(hiddenDate));
+                while (!allGeocodes.isEmpty()) {
+                    int cachesInBatch = 0;
+                    HashSet<String> geocodesOfBatch = new HashSet<String>(CACHES_PER_BATCH);
+                    for (Iterator<String> iterator = allGeocodes.iterator(); iterator.hasNext() && cachesInBatch < CACHES_PER_BATCH;) {
+                        final String geocode = iterator.next();
+                        geocodesOfBatch.add(geocode);
+                        cachesInBatch++;
                     }
-
-                    XmlUtils.multipleTexts(gpx, PREFIX_GPX,
-                            "name", cache.getGeocode(),
-                            "desc", cache.getName(),
-                            "url", cache.getUrl(),
-                            "urlname", cache.getName(),
-                            "sym", cache.isFound() ? "Geocache Found" : "Geocache",
-                            "type", "Geocache|" + cache.getType().pattern);
-
-                    gpx.startTag(PREFIX_GROUNDSPEAK, "cache");
-                    gpx.attribute("", "id", cache.getCacheId());
-                    gpx.attribute("", "available", !cache.isDisabled() ? "True" : "False");
-                    gpx.attribute("", "archives", cache.isArchived() ? "True" : "False");
-
-                    XmlUtils.multipleTexts(gpx, PREFIX_GROUNDSPEAK,
-                            "name", cache.getName(),
-                            "placed_by", cache.getOwnerDisplayName(),
-                            "owner", cache.getOwnerUserId(),
-                            "type", cache.getType().pattern,
-                            "container", cache.getSize().id,
-                            "difficulty", Float.toString(cache.getDifficulty()),
-                            "terrain", Float.toString(cache.getTerrain()),
-                            "country", cache.getLocation(),
-                            "state", "",
-                            "encoded_hints", cache.getHint());
-
-                    writeAttributes(gpx, cache);
-
-                    gpx.startTag(PREFIX_GROUNDSPEAK, "short_description");
-                    gpx.attribute("", "html", BaseUtils.containsHtml(cache.getShortDescription()) ? "True" : "False");
-                    gpx.text(cache.getShortDescription());
-                    gpx.endTag(PREFIX_GROUNDSPEAK, "short_description");
-
-                    gpx.startTag(PREFIX_GROUNDSPEAK, "long_description");
-                    gpx.attribute("", "html", BaseUtils.containsHtml(cache.getDescription()) ? "True" : "False");
-                    gpx.text(cache.getDescription());
-                    gpx.endTag(PREFIX_GROUNDSPEAK, "long_description");
-
-                    writeLogs(gpx, cache);
-
-                    gpx.endTag(PREFIX_GROUNDSPEAK, "cache");
-                    gpx.endTag(PREFIX_GPX, "wpt");
-
-                    writeWaypoints(gpx, cache);
-
-                    publishProgress(i + 1);
+                    allGeocodes.removeAll(geocodesOfBatch);
+                    exportBatch(gpx, geocodesOfBatch);
                 }
 
                 gpx.endTag(PREFIX_GPX, "gpx");
                 gpx.endDocument();
-            } catch (final IOException e) {
+            } catch (final Exception e) {
                 Log.e("GpxExport.ExportTask export", e);
 
                 if (writer != null) {
@@ -226,6 +195,67 @@ class GpxExport extends AbstractExport {
             return exportFile;
         }
 
+        private void exportBatch(final XmlSerializer gpx, HashSet<String> geocodesOfBatch) throws IOException {
+            Set<Geocache> caches = cgData.loadCaches(geocodesOfBatch, LoadFlags.LOAD_ALL_DB_ONLY);
+            for (Geocache cache : caches) {
+                gpx.startTag(PREFIX_GPX, "wpt");
+                gpx.attribute("", "lat", Double.toString(cache.getCoords().getLatitude()));
+                gpx.attribute("", "lon", Double.toString(cache.getCoords().getLongitude()));
+
+                final Date hiddenDate = cache.getHiddenDate();
+                if (hiddenDate != null) {
+                    XmlUtils.simpleText(gpx, PREFIX_GPX, "time", dateFormatZ.format(hiddenDate));
+                }
+
+                XmlUtils.multipleTexts(gpx, PREFIX_GPX,
+                        "name", cache.getGeocode(),
+                        "desc", cache.getName(),
+                        "url", cache.getUrl(),
+                        "urlname", cache.getName(),
+                        "sym", cache.isFound() ? "Geocache Found" : "Geocache",
+                        "type", "Geocache|" + cache.getType().pattern);
+
+                gpx.startTag(PREFIX_GROUNDSPEAK, "cache");
+                gpx.attribute("", "id", cache.getCacheId());
+                gpx.attribute("", "available", !cache.isDisabled() ? "True" : "False");
+                gpx.attribute("", "archives", cache.isArchived() ? "True" : "False");
+
+                XmlUtils.multipleTexts(gpx, PREFIX_GROUNDSPEAK,
+                        "name", cache.getName(),
+                        "placed_by", cache.getOwnerDisplayName(),
+                        "owner", cache.getOwnerUserId(),
+                        "type", cache.getType().pattern,
+                        "container", cache.getSize().id,
+                        "difficulty", Float.toString(cache.getDifficulty()),
+                        "terrain", Float.toString(cache.getTerrain()),
+                        "country", cache.getLocation(),
+                        "state", "",
+                        "encoded_hints", cache.getHint());
+
+                writeAttributes(gpx, cache);
+
+                gpx.startTag(PREFIX_GROUNDSPEAK, "short_description");
+                gpx.attribute("", "html", BaseUtils.containsHtml(cache.getShortDescription()) ? "True" : "False");
+                gpx.text(cache.getShortDescription());
+                gpx.endTag(PREFIX_GROUNDSPEAK, "short_description");
+
+                gpx.startTag(PREFIX_GROUNDSPEAK, "long_description");
+                gpx.attribute("", "html", BaseUtils.containsHtml(cache.getDescription()) ? "True" : "False");
+                gpx.text(cache.getDescription());
+                gpx.endTag(PREFIX_GROUNDSPEAK, "long_description");
+
+                writeLogs(gpx, cache);
+
+                gpx.endTag(PREFIX_GROUNDSPEAK, "cache");
+                gpx.endTag(PREFIX_GPX, "wpt");
+
+                writeWaypoints(gpx, cache);
+
+                countExported++;
+                publishProgress(countExported);
+            }
+        }
+
         private void writeWaypoints(final XmlSerializer gpx, final Geocache cache) throws IOException {
             List<Waypoint> waypoints = cache.getWaypoints();
             List<Waypoint> ownWaypoints = new ArrayList<Waypoint>(waypoints.size());

refactoring: improve performance of GPX export