diff options
author | Christoph Berg | 2020-03-24 20:53:17 +0000 |
---|---|---|
committer | Christoph Berg | 2020-03-24 20:53:17 +0000 |
commit | 0494405e4adb03a6e264c5ba501ea6985100a5bc (patch) | |
tree | 4adddf9e49542f3dc85e99c9861ad9051397bb74 | |
parent | 70ae7e285f7e58b9f7b7271bf88ab56dfa816389 (diff) |
Generate pool/main/p/package/index.html files
-rw-r--r-- | pgapt-db/sql/pgdg_apt.sql | 9 | ||||
-rwxr-xr-x | repo/bin/generate-archive-lists | 72 | ||||
-rwxr-xr-x | repo/bin/import-packagesfile | 5 | ||||
-rwxr-xr-x | repo/bin/import-sourcesfile | 4 |
4 files changed, 71 insertions, 19 deletions
diff --git a/pgapt-db/sql/pgdg_apt.sql b/pgapt-db/sql/pgdg_apt.sql index 78a8c3f..7852204 100644 --- a/pgapt-db/sql/pgdg_apt.sql +++ b/pgapt-db/sql/pgdg_apt.sql @@ -56,8 +56,13 @@ COMMENT ON TABLE distribution IS 'known distributions by Packages files'; -- PACKAGE DATA +CREATE TABLE src ( + source text PRIMARY KEY, + upload boolean -- per-package index.html upload status: null = to do, t = done, f = error +); + CREATE TABLE source ( - source text NOT NULL, + source text NOT NULL REFERENCES src, srcversion debversion NOT NULL, control text NOT NULL, c jsonb, @@ -90,7 +95,7 @@ CREATE TABLE package ( REFERENCES architecture (architecture), control text NOT NULL, c jsonb, - source text NOT NULL, + source text NOT NULL REFERENCES src, srcversion debversion NOT NULL, time timestamptz(0) NOT NULL, upload boolean, -- null = to do, t = done, f = error diff --git a/repo/bin/generate-archive-lists b/repo/bin/generate-archive-lists index b14474c..f1986d7 100755 --- a/repo/bin/generate-archive-lists +++ b/repo/bin/generate-archive-lists @@ -12,9 +12,10 @@ cd /srv/apt/archive/pub/repos/apt export PGSERVICE=pgapt -DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution ORDER BY 1, 2;" -DISTS="SELECT distinct distribution FROM srcdistribution ORDER BY 1;" +DIST_ARCHS="SELECT distinct distribution, architecture FROM distribution WHERE last_update > now() - '2 days'::interval ORDER BY 1, 2;" +DISTS="SELECT distinct distribution FROM srcdistribution WHERE last_update > now() - '2 days'::interval ORDER BY 1;" +# generate Packages files psql -AXt -F ' ' -c "$DIST_ARCHS" | \ while read distribution architecture; do [ -t 1 ] && echo "$distribution-archive/$architecture ..." @@ -26,6 +27,7 @@ while read distribution architecture; do bzip2 --keep $DIR/Packages done +# generate Sources and InRelease files psql -AXt -F ' ' -c "$DISTS" | \ while read distribution; do [ -t 1 ] && echo "$distribution-archive/source ..." @@ -61,21 +63,57 @@ done indexhtml dists -DIRECTORIES="SELECT DISTINCT directory FROM sourcefile;" -psql -AXtc "$DIRECTORIES" | while read dir; do - mkdir -p $dir -done +# upload to S3 and invalidate cache for the dists/ dir +aws s3 sync --quiet /srv/apt/archive/pub/repos/apt/dists $S3/dists +aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*" > /dev/null -for dir in pool/*/*/; do - indexhtml $dir -done -for dir in pool/*/; do - indexhtml $dir -done -indexhtml pool +cd /srv/apt/archive/pub/repos/apt -indexhtml . +# generate index files for package pool directories +PACKAGES=$(psql -XAtc "SELECT source FROM src WHERE upload IS NULL") +for PACKAGE in $PACKAGES; do + case $PACKAGE in + lib*) INITIAL="$(echo $PACKAGE | cut -c 1-4)" ;; + *) INITIAL="$(echo $PACKAGE | cut -c 1)" ;; + esac + DIRECTORY="pool/main/$INITIAL/$PACKAGE" + QUERY="SELECT sf.filename, NULL::text AS size, s.time + FROM source s + JOIN sourcefile sf ON (s.source, s.srcversion) = (sf.source, sf.srcversion) + WHERE s.source = '$PACKAGE' + UNION ALL SELECT regexp_replace(p.c->>'filename', '.*/', ''), p.c->>'size', p.time + FROM package p + WHERE p.source = '$PACKAGE' + ORDER BY filename" + FORMAT="WITH q AS ($QUERY) + SELECT format(E'<a href=\"%s\">%s</a> %s %s <br /> \\n', + regexp_replace(filename, '\\+', '%2B', 'g'), + filename, + time, + size || ' bytes') + FROM q" -# upload to S3 and invalidate cache for the dists/ dir -aws s3 sync /srv/apt/archive/pub/repos/apt/dists $S3/dists -aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/dists/*" + mkdir -p "$DIRECTORY" + + ( + echo "<tt>" + echo "<a href=\"../index.html\">../</a> <br />" + psql -XAtc "$FORMAT" + echo "</tt>" + ) > "$DIRECTORY/index.html" + + upload-to-s3 "$DIRECTORY/index.html" + aws cloudfront create-invalidation --distribution-id E2CPN16I5GL3S7 --path "/pub/repos/apt/$DIRECTORY/index.html" > /dev/null + + indexhtml "pool/main/$INITIAL" + upload-to-s3 "pool/main/$INITIAL/index.html" + + psql -Xqc "UPDATE src SET upload = true WHERE src = '$PACKAGE'" +done + +# alert about any failed uploads +psql -XAt <<EOF +SELECT 'failed sourcefile upload:', * FROM sourcefile WHERE upload IS FALSE; +SELECT 'failed package upload:', * FROM package WHERE upload IS FALSE; +SELECT 'failed pool/main/p/package/index.html upload:', * FROM src WHERE upload IS FALSE; +EOF diff --git a/repo/bin/import-packagesfile b/repo/bin/import-packagesfile index d5c0fb1..1af12b1 100755 --- a/repo/bin/import-packagesfile +++ b/repo/bin/import-packagesfile @@ -80,6 +80,11 @@ def parseFile(distribution, component, architecture, packagesfile, ctime): package, version, arch)) source_version = binnmu_re.sub('', source_version) + cur.execute("""INSERT INTO src (source) VALUES (%s) + ON CONFLICT (source) + DO UPDATE SET upload = NULL""", + [source]) + # # find link to source package # cur.execute("""SELECT 1 FROM source # WHERE (source, srcversion) = (%s, %s)""", diff --git a/repo/bin/import-sourcesfile b/repo/bin/import-sourcesfile index d41f24a..3ff0c33 100755 --- a/repo/bin/import-sourcesfile +++ b/repo/bin/import-sourcesfile @@ -60,6 +60,10 @@ def parseSourceFile(distribution, component, packagesfile, ctime): [package, version]) found = cur.fetchone() if not found: # this is the first time we see this package + cur.execute("""INSERT INTO src (source) VALUES (%s) + ON CONFLICT (source) + DO UPDATE SET upload = NULL""", + [package]) cur.execute("""INSERT INTO source (source, srcversion, control, c, time) VALUES (%s, %s, %s, control2jsonb(%s), %s)""", [package, version, control, control, ctime]) |