Upload sourcefiles to S3

4 files changed, 53 insertions, 90 deletions

diff --git a/pgapt-db/sql/pgdg_apt.sql b/pgapt-db/sql/pgdg_apt.sql
index 6911dac..fc19487 100644
--- a/pgapt-db/sql/pgdg_apt.sql
+++ b/pgapt-db/sql/pgdg_apt.sql
@@ -170,5 +170,9 @@ COMMENT ON TABLE packagehist IS 'current and historic Packages files';
 
 GRANT USAGE ON SCHEMA apt TO PUBLIC;
 GRANT SELECT ON ALL TABLES IN SCHEMA apt TO PUBLIC;
+GRANT INSERT ON ALL TABLES IN SCHEMA apt TO aptuser;
+GRANT DELETE ON sourcelist, packagelist TO aptuser;
+GRANT UPDATE (last_update) ON srcdistribution, distribution TO aptuser;
+GRANT UPDATE (upload) ON sourcefile, package TO aptuser;
 
 COMMIT;
diff --git a/repo/bin/morguefiles-to-s3 b/repo/bin/morguefiles-to-s3
deleted file mode 100755
index 2e0d67e..0000000
--- a/repo/bin/morguefiles-to-s3
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-# directory where we keep metadata of morgue files
-MORGUE=/srv/apt/morgue
-MORGUEINDICES="$MORGUE/index.queue"
-# S3 bucket
-S3="s3://apt-archive.postgresql.org"
-export TZ=UTC
-
-cd /srv/apt/repo/morgue
-
-for file in *.deb *.dsc *.diff.gz *.tar.* *.buildinfo; do
-  [ -f "$file" ] || continue
-
-  # determine source name
-  case $file in
-    *.deb)
-      source=$(dpkg-deb -I $file control | awk '/^Source:/ { print $2 }')
-      [ -z "$source" ] && source=${file%%_*}
-      ;;
-    *)
-      source=${file%%_*}
-      ;;
-  esac
-  [ "$file" != "$source" ]
-
-  # record meta data
-  initial=$(echo $source | egrep -o '^(lib)?.')
-  dir="$MORGUE/$initial/$source"
-  mkdir -p "$dir"
-  stat -c '%s' "$file" > "$dir/$file.size"
-  touch -r "$file" "$dir/$file.size"
-  case $file in
-    *.deb)
-      dpkg-deb -I $file control > "$dir/$file.info"
-      ;;
-  esac
-
-  # remember filenames for index.html generation
-  echo "$initial/$source" >> $MORGUEINDICES
-  echo "$initial" >> $MORGUEINDICES
-  echo "/" >> $MORGUEINDICES
-
-  # move file to S3
-  aws s3 cp "$file" "$S3/$initial/$source/$file"
-  mkdir -p "$MORGUE/done"
-  mv "$file" "$MORGUE/done"
-done
-
-if [ -s "$MORGUEINDICES" ]; then
-#  # build new indices
-#  apt-ftparchive -qq --db ../apt-ftparchive/packages.db packages . | xz > .Packages.xz.new
-#  apt-ftparchive -qq --db ../apt-ftparchive/sources.db sources . 2>/dev/null | xz > .Sources.xz.new # suppress warnings about missing .orig.tar files etc.
-#  mv .Packages.xz.new Packages.xz
-#  mv .Sources.xz.new Sources.xz
-#  # build new release
-#  rm -f Release
-#  apt-ftparchive -qq release . > .Release.new
-#  mv .Release.new Release
-
-  for dir in $(sort -u $MORGUEINDICES); do
-    echo "Generating index.html for $dir ..."
-    cd "$MORGUE/$dir"
-
-    echo "<table border=\"0\">" > index.html
-
-    echo "<tr><td><a href=\"../\">../</a></td></tr>" >> index.html
-    for d in */; do
-      [ -d "$d" ] || continue
-      echo "<tr><td><a href=\"$d\">$d</a></td></tr>"
-    done >> index.html
-
-    for sizefile in *.size; do
-      [ -f "$sizefile" ] || continue
-      file="${sizefile%.size}"
-      echo "<tr><td><a href=\"$file\">$file</a></td> <td>$(cat $sizefile) bytes</td> <td>$(stat -c %y $sizefile)</td></tr>"
-    done >> index.html
-
-    echo "</tr>" >> index.html
-
-    s3file="$S3/$dir/index.html"
-    [ "$dir" = "/" ] && s3file="$S3/index.html"
-    aws s3 cp index.html "$s3file"
-  done
-
-fi
-
-rm -f "$MORGUEINDICES"
diff --git a/repo/bin/upload-poolfiles b/repo/bin/upload-poolfiles
new file mode 100755
index 0000000..f5d0bf5
--- /dev/null
+++ b/repo/bin/upload-poolfiles
@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+
+import psycopg2, subprocess, sys
+
+pg = psycopg2.connect('service=pgapt')
+cur = pg.cursor()
+
+while True:
+    cur.execute("""SELECT directory, filename FROM sourcefile WHERE upload IS NULL LIMIT 1 FOR UPDATE SKIP LOCKED""")
+    sourcefile = cur.fetchone()
+    if not sourcefile: # we are done
+        sys.exit(0)
+
+    path = "%s/%s" % (sourcefile[0], sourcefile[1])
+    filename = sourcefile[1]
+
+    print(path, '...')
+
+    success = subprocess.call(['bin/upload-to-s3', path])
+    if success == 0:
+        upload = 't'
+    else:
+        upload = 'f'
+
+    cur.execute("""UPDATE sourcefile SET upload = %s WHERE filename = %s""",
+            [upload, filename])
+
+    pg.commit()
diff --git a/repo/bin/upload-to-s3 b/repo/bin/upload-to-s3
new file mode 100755
index 0000000..db661ca
--- /dev/null
+++ b/repo/bin/upload-to-s3
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# S3 bucket
+S3="s3://apt-archive.postgresql.org"
+
+set -eu
+
+cd $(dirname $0)/..
+
+FILE="$1"
+test -f "$FILE"
+
+# upload to S3
+aws s3 cp "$FILE" "$S3/$FILE"
+
+# try if we can get the file back
+RESTORE=$(mktemp --tmpdir upload-to-s3.XXXXXX)
+trap "rm -f $RESTORE" EXIT
+aws s3 cp "$S3/$FILE" "$RESTORE"
+
+cmp "$FILE" "$RESTORE"