Generate pool/main/p/package/index.html files

4 files changed, 71 insertions, 19 deletions

diff --git a/pgapt-db/sql/pgdg_apt.sql b/pgapt-db/sql/pgdg_apt.sql
index 78a8c3f..7852204 100644
--- a/pgapt-db/sql/pgdg_apt.sql
+++ b/pgapt-db/sql/pgdg_apt.sql
@@ -56,8 +56,13 @@ COMMENT ON TABLE distribution IS 'known distributions by Packages files';
 
 -- PACKAGE DATA
 
+CREATE TABLE src (
+	source text PRIMARY KEY,
+	upload boolean -- per-package index.html upload status: null = to do, t = done, f = error
+);
+
 CREATE TABLE source (
-	source text NOT NULL,
+	source text NOT NULL REFERENCES src,
 	srcversion debversion NOT NULL,
 	control text NOT NULL,
 	c jsonb,
@@ -90,7 +95,7 @@ CREATE TABLE package (
 		REFERENCES architecture (architecture),
 	control text NOT NULL,
 	c jsonb,
-	source text NOT NULL,
+	source text NOT NULL REFERENCES src,
 	srcversion debversion NOT NULL,
 	time timestamptz(0) NOT NULL,
 	upload boolean, -- null = to do, t = done, f = error
diff --git a/repo/bin/generate-archive-lists b/repo/bin/generate-archive-lists
index b14474c..f1986d7 100755
--- a/repo/bin/generate-archive-lists
+++ b/repo/bin/generate-archive-lists
@@ -12,9 +12,10 @@ cd /srv/apt/archive/pub/repos/apt
 
 export PGSERVICE=pgapt
 
-DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution ORDER BY 1, 2;"
-DISTS="SELECT distinct distribution FROM srcdistribution ORDER BY 1;"
+DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution WHERE last_update > now() - '2 days'::interval ORDER BY 1, 2;"
+DISTS="SELECT distinct distribution FROM srcdistribution WHERE last_update > now() - '2 days'::interval ORDER BY 1;"
 
+# generate Packages files
 psql -AXt -F ' ' -c "$DIST_ARCHS" | \
 while read distribution architecture; do
   [ -t 1 ] && echo "$distribution-archive/$architecture ..."
@@ -26,6 +27,7 @@ while read distribution architecture; do
   bzip2 --keep $DIR/Packages
 done
 
+# generate Sources and InRelease files
 psql -AXt -F ' ' -c "$DISTS" | \
 while read distribution; do
   [ -t 1 ] && echo "$distribution-archive/source ..."
@@ -61,21 +63,57 @@ done
 
 indexhtml dists
 
-DIRECTORIES="SELECT DISTINCT directory FROM sourcefile;"
-psql -AXtc "$DIRECTORIES" | while read dir; do
-  mkdir -p $dir
-done
+# upload to S3 and invalidate cache for the dists/ dir
+aws s3 sync --quiet /srv/apt/archive/pub/repos/apt/dists $S3/dists
+aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*" > /dev/null
 
-for dir in pool/*/*/; do
-  indexhtml $dir
-done
-for dir in pool/*/; do
-  indexhtml $dir
-done
-indexhtml pool
+cd /srv/apt/archive/pub/repos/apt
 
-indexhtml .
+# generate index files for package pool directories
+PACKAGES=$(psql -XAtc "SELECT source FROM src WHERE upload IS NULL")
+for PACKAGE in $PACKAGES; do
+  case $PACKAGE in
+    lib*) INITIAL="$(echo $PACKAGE | cut -c 1-4)" ;;
+    *)    INITIAL="$(echo $PACKAGE | cut -c 1)" ;;
+  esac
+  DIRECTORY="pool/main/$INITIAL/$PACKAGE"
+  QUERY="SELECT sf.filename, NULL::text AS size, s.time
+    FROM source s
+    JOIN sourcefile sf ON (s.source, s.srcversion) = (sf.source, sf.srcversion)
+    WHERE s.source = '$PACKAGE'
+  UNION ALL SELECT regexp_replace(p.c->>'filename', '.*/', ''), p.c->>'size', p.time
+    FROM package p
+    WHERE p.source = '$PACKAGE'
+  ORDER BY filename"
+  FORMAT="WITH q AS ($QUERY)
+    SELECT format(E'<a href=\"%s\">%s</a> %s %s <br /> \\n',
+      regexp_replace(filename, '\\+', '%2B', 'g'),
+      filename,
+      time,
+      size || ' bytes')
+    FROM q"
 
-# upload to S3 and invalidate cache for the dists/ dir
-aws s3 sync /srv/apt/archive/pub/repos/apt/dists $S3/dists
-aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*"
+  mkdir -p "$DIRECTORY"
+
+  (
+    echo "<tt>"
+    echo "<a href=\"../index.html\">../</a> <br />"
+    psql -XAtc "$FORMAT"
+    echo "</tt>"
+  ) > "$DIRECTORY/index.html"
+
+  upload-to-s3 "$DIRECTORY/index.html"
+  aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/$DIRECTORY/index.html" > /dev/null
+
+  indexhtml "pool/main/$INITIAL"
+  upload-to-s3 "pool/main/$INITIAL/index.html"
+
+  psql -Xqc "UPDATE src SET upload = true WHERE src = '$PACKAGE'"
+done
+
+# alert about any failed uploads
+psql -XAt <<EOF
+SELECT 'failed sourcefile upload:', * FROM sourcefile WHERE upload IS FALSE;
+SELECT 'failed package upload:', * FROM package WHERE upload IS FALSE;
+SELECT 'failed pool/main/p/package/index.html upload:', * FROM src WHERE upload IS FALSE;
+EOF
diff --git a/repo/bin/import-packagesfile b/repo/bin/import-packagesfile
index d5c0fb1..1af12b1 100755
--- a/repo/bin/import-packagesfile
+++ b/repo/bin/import-packagesfile
@@ -80,6 +80,11 @@ def parseFile(distribution, component, architecture, packagesfile, ctime):
                             package, version, arch))
                 source_version = binnmu_re.sub('', source_version)
 
+            cur.execute("""INSERT INTO src (source) VALUES (%s)
+                           ON CONFLICT (source)
+                           DO UPDATE SET upload = NULL""",
+                           [source])
+
 #            # find link to source package
 #            cur.execute("""SELECT 1 FROM source
 #                           WHERE (source, srcversion) = (%s, %s)""",
diff --git a/repo/bin/import-sourcesfile b/repo/bin/import-sourcesfile
index d41f24a..3ff0c33 100755
--- a/repo/bin/import-sourcesfile
+++ b/repo/bin/import-sourcesfile
@@ -60,6 +60,10 @@ def parseSourceFile(distribution, component, packagesfile, ctime):
                        [package, version])
         found = cur.fetchone()
         if not found: # this is the first time we see this package
+            cur.execute("""INSERT INTO src (source) VALUES (%s)
+                           ON CONFLICT (source)
+                           DO UPDATE SET upload = NULL""",
+                           [package])
             cur.execute("""INSERT INTO source (source, srcversion, control, c, time)
                            VALUES (%s, %s, %s, control2jsonb(%s), %s)""",
                            [package, version, control, control, ctime])