summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristoph Berg2020-03-24 20:53:17 +0000
committerChristoph Berg2020-03-24 20:53:17 +0000
commit0494405e4adb03a6e264c5ba501ea6985100a5bc (patch)
tree4adddf9e49542f3dc85e99c9861ad9051397bb74
parent70ae7e285f7e58b9f7b7271bf88ab56dfa816389 (diff)
Generate pool/main/p/package/index.html files
-rw-r--r--pgapt-db/sql/pgdg_apt.sql9
-rwxr-xr-xrepo/bin/generate-archive-lists72
-rwxr-xr-xrepo/bin/import-packagesfile5
-rwxr-xr-xrepo/bin/import-sourcesfile4
4 files changed, 71 insertions, 19 deletions
diff --git a/pgapt-db/sql/pgdg_apt.sql b/pgapt-db/sql/pgdg_apt.sql
index 78a8c3f..7852204 100644
--- a/pgapt-db/sql/pgdg_apt.sql
+++ b/pgapt-db/sql/pgdg_apt.sql
@@ -56,8 +56,13 @@ COMMENT ON TABLE distribution IS 'known distributions by Packages files';
-- PACKAGE DATA
+CREATE TABLE src (
+ source text PRIMARY KEY,
+ upload boolean -- per-package index.html upload status: null = to do, t = done, f = error
+);
+
CREATE TABLE source (
- source text NOT NULL,
+ source text NOT NULL REFERENCES src,
srcversion debversion NOT NULL,
control text NOT NULL,
c jsonb,
@@ -90,7 +95,7 @@ CREATE TABLE package (
REFERENCES architecture (architecture),
control text NOT NULL,
c jsonb,
- source text NOT NULL,
+ source text NOT NULL REFERENCES src,
srcversion debversion NOT NULL,
time timestamptz(0) NOT NULL,
upload boolean, -- null = to do, t = done, f = error
diff --git a/repo/bin/generate-archive-lists b/repo/bin/generate-archive-lists
index b14474c..f1986d7 100755
--- a/repo/bin/generate-archive-lists
+++ b/repo/bin/generate-archive-lists
@@ -12,9 +12,10 @@ cd /srv/apt/archive/pub/repos/apt
export PGSERVICE=pgapt
-DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution ORDER BY 1, 2;"
-DISTS="SELECT distinct distribution FROM srcdistribution ORDER BY 1;"
+DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution WHERE last_update > now() - '2 days'::interval ORDER BY 1, 2;"
+DISTS="SELECT distinct distribution FROM srcdistribution WHERE last_update > now() - '2 days'::interval ORDER BY 1;"
+# generate Packages files
psql -AXt -F ' ' -c "$DIST_ARCHS" | \
while read distribution architecture; do
[ -t 1 ] && echo "$distribution-archive/$architecture ..."
@@ -26,6 +27,7 @@ while read distribution architecture; do
bzip2 --keep $DIR/Packages
done
+# generate Sources and InRelease files
psql -AXt -F ' ' -c "$DISTS" | \
while read distribution; do
[ -t 1 ] && echo "$distribution-archive/source ..."
@@ -61,21 +63,57 @@ done
indexhtml dists
-DIRECTORIES="SELECT DISTINCT directory FROM sourcefile;"
-psql -AXtc "$DIRECTORIES" | while read dir; do
- mkdir -p $dir
-done
+# upload to S3 and invalidate cache for the dists/ dir
+aws s3 sync --quiet /srv/apt/archive/pub/repos/apt/dists $S3/dists
+aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*" > /dev/null
-for dir in pool/*/*/; do
- indexhtml $dir
-done
-for dir in pool/*/; do
- indexhtml $dir
-done
-indexhtml pool
+cd /srv/apt/archive/pub/repos/apt
-indexhtml .
+# generate index files for package pool directories
+PACKAGES=$(psql -XAtc "SELECT source FROM src WHERE upload IS NULL")
+for PACKAGE in $PACKAGES; do
+ case $PACKAGE in
+ lib*) INITIAL="$(echo $PACKAGE | cut -c 1-4)" ;;
+ *) INITIAL="$(echo $PACKAGE | cut -c 1)" ;;
+ esac
+ DIRECTORY="pool/main/$INITIAL/$PACKAGE"
+ QUERY="SELECT sf.filename, NULL::text AS size, s.time
+ FROM source s
+ JOIN sourcefile sf ON (s.source, s.srcversion) = (sf.source, sf.srcversion)
+ WHERE s.source = '$PACKAGE'
+ UNION ALL SELECT regexp_replace(p.c->>'filename', '.*/', ''), p.c->>'size', p.time
+ FROM package p
+ WHERE p.source = '$PACKAGE'
+ ORDER BY filename"
+ FORMAT="WITH q AS ($QUERY)
+ SELECT format(E'<a href=\"%s\">%s</a> %s %s <br /> \\n',
+ regexp_replace(filename, '\\+', '%2B', 'g'),
+ filename,
+ time,
+ size || ' bytes')
+ FROM q"
-# upload to S3 and invalidate cache for the dists/ dir
-aws s3 sync /srv/apt/archive/pub/repos/apt/dists $S3/dists
-aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*"
+ mkdir -p "$DIRECTORY"
+
+ (
+ echo "<tt>"
+ echo "<a href=\"../index.html\">../</a> <br />"
+ psql -XAtc "$FORMAT"
+ echo "</tt>"
+ ) > "$DIRECTORY/index.html"
+
+ upload-to-s3 "$DIRECTORY/index.html"
+ aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/$DIRECTORY/index.html" > /dev/null
+
+ indexhtml "pool/main/$INITIAL"
+ upload-to-s3 "pool/main/$INITIAL/index.html"
+
+ psql -Xqc "UPDATE src SET upload = true WHERE src = '$PACKAGE'"
+done
+
+# alert about any failed uploads
+psql -XAt <<EOF
+SELECT 'failed sourcefile upload:', * FROM sourcefile WHERE upload IS FALSE;
+SELECT 'failed package upload:', * FROM package WHERE upload IS FALSE;
+SELECT 'failed pool/main/p/package/index.html upload:', * FROM src WHERE upload IS FALSE;
+EOF
diff --git a/repo/bin/import-packagesfile b/repo/bin/import-packagesfile
index d5c0fb1..1af12b1 100755
--- a/repo/bin/import-packagesfile
+++ b/repo/bin/import-packagesfile
@@ -80,6 +80,11 @@ def parseFile(distribution, component, architecture, packagesfile, ctime):
package, version, arch))
source_version = binnmu_re.sub('', source_version)
+ cur.execute("""INSERT INTO src (source) VALUES (%s)
+ ON CONFLICT (source)
+ DO UPDATE SET upload = NULL""",
+ [source])
+
# # find link to source package
# cur.execute("""SELECT 1 FROM source
# WHERE (source, srcversion) = (%s, %s)""",
diff --git a/repo/bin/import-sourcesfile b/repo/bin/import-sourcesfile
index d41f24a..3ff0c33 100755
--- a/repo/bin/import-sourcesfile
+++ b/repo/bin/import-sourcesfile
@@ -60,6 +60,10 @@ def parseSourceFile(distribution, component, packagesfile, ctime):
[package, version])
found = cur.fetchone()
if not found: # this is the first time we see this package
+ cur.execute("""INSERT INTO src (source) VALUES (%s)
+ ON CONFLICT (source)
+ DO UPDATE SET upload = NULL""",
+ [package])
cur.execute("""INSERT INTO source (source, srcversion, control, c, time)
VALUES (%s, %s, %s, control2jsonb(%s), %s)""",
[package, version, control, control, ctime])