Skip to content

Commit 8be65e0

Browse files
jbajba
committed
internal/postgres: add ReInsertLatestVersion
New latest-version info can retract the current latest good version, resulting in an older version becoming the new latest good version. Add a function that updates the search_documents table when that happens. A later CL will also update imports_unique, and will call this function from the worker. For golang/go#44710 Change-Id: I09569e27d384ed7defb06a4e81facdb96c22caeb Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/303312 Trust: Jonathan Amsterdam <[email protected]> Run-TryBot: Jonathan Amsterdam <[email protected]> Reviewed-by: Jamal Carvalho <[email protected]> Reviewed-by: Julie Qiu <[email protected]>
1 parent 2fd373d commit 8be65e0

File tree

4 files changed

+162
-6
lines changed

4 files changed

+162
-6
lines changed

‎internal/postgres/insert_module.go

+89
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,95 @@ func insertReadmes(ctx context.Context, db *database.DB,
632632
return db.BulkUpsert(ctx, "readmes", readmeCols, readmeValues, []string{"unit_id"})
633633
}
634634

635+
// ReInsertLatestVersion checks that the latest good version matches the version
636+
// in search_documents. If it doesn't, it inserts the latest good version into
637+
// search_documents and imports_unique.
638+
func (db *DB) ReInsertLatestVersion(ctx context.Context, modulePath string) (err error) {
639+
defer derrors.WrapStack(&err, "ReInsertLatestVersion(%q)", modulePath)
640+
641+
return db.db.Transact(ctx, sql.LevelRepeatableRead, func(tx *database.DB) error {
642+
// Hold the lock on the module path throughout.
643+
if err := lock(ctx, tx, modulePath); err != nil {
644+
return err
645+
}
646+
647+
lmv, _, err := getLatestModuleVersions(ctx, tx, modulePath)
648+
if err != nil {
649+
return err
650+
}
651+
if lmv.GoodVersion == "" {
652+
// TODO(golang/go#44710): once we are confident that
653+
// latest_module_versions is accurate and up to date, we can assume
654+
// that a missing GoodVersion should mean that there are no good
655+
// versions remaining, and we should remove the current module from
656+
// search_documents.
657+
log.Debugf(ctx, "ReInsertLatestVersion(%q): no good version", modulePath)
658+
return nil
659+
}
660+
// Is the latest good version in search_documents?
661+
var x int
662+
switch err := tx.QueryRow(ctx, `
663+
SELECT 1
664+
FROM search_documents
665+
WHERE module_path = $1
666+
AND version = $2
667+
`, modulePath, lmv.GoodVersion).Scan(&x); err {
668+
case sql.ErrNoRows:
669+
break
670+
case nil:
671+
log.Debugf(ctx, "ReInsertLatestVersion(%q): good version %s found in search_documents; doing nothing",
672+
modulePath, lmv.GoodVersion)
673+
return nil
674+
default:
675+
return err
676+
}
677+
678+
// The latest good version is not in search_documents. Is this an
679+
// alternative module path?
680+
alt, err := isAlternativeModulePath(ctx, tx, modulePath)
681+
if err != nil {
682+
return err
683+
}
684+
if alt {
685+
log.Debugf(ctx, "ReInsertLatestVersion(%q): alternative module path; doing nothing", modulePath)
686+
return nil
687+
}
688+
689+
// Not an alternative module path. Read the module information at the
690+
// latest good version.
691+
pkgMetas, err := getPackagesInUnit(ctx, tx, modulePath, modulePath, lmv.GoodVersion, db.bypassLicenseCheck)
692+
if err != nil {
693+
return err
694+
}
695+
// We only need the readme for the module.
696+
readme, err := getModuleReadme(ctx, tx, modulePath, lmv.GoodVersion)
697+
if err != nil {
698+
return err
699+
}
700+
// Insert into search_documents.
701+
for _, pkg := range pkgMetas {
702+
if isInternalPackage(pkg.Path) {
703+
continue
704+
}
705+
args := UpsertSearchDocumentArgs{
706+
PackagePath: pkg.Path,
707+
ModulePath: modulePath,
708+
Version: lmv.GoodVersion,
709+
Synopsis: pkg.Synopsis,
710+
}
711+
if pkg.Path == modulePath && readme != nil {
712+
args.ReadmeFilePath = readme.Filepath
713+
args.ReadmeContents = readme.Contents
714+
}
715+
if err := UpsertSearchDocument(ctx, tx, args); err != nil {
716+
return err
717+
}
718+
}
719+
log.Debugf(ctx, "ReInsertLatestVersion(%q): re-inserted at latest good version %s", modulePath, lmv.GoodVersion)
720+
return nil
721+
})
722+
}
723+
635724
// lock obtains an exclusive, transaction-scoped advisory lock on modulePath.
636725
func lock(ctx context.Context, tx *database.DB, modulePath string) (err error) {
637726
defer derrors.WrapStack(&err, "lock(%s)", modulePath)

‎internal/postgres/insert_module_test.go

+58
Original file line numberDiff line numberDiff line change
@@ -651,3 +651,61 @@ func TestIsAlternativeModulePath(t *testing.T) {
651651
}
652652
}
653653
}
654+
655+
func TestReInsertLatestVersion(t *testing.T) {
656+
t.Parallel()
657+
ctx := context.Background()
658+
testDB, release := acquire(t)
659+
defer release()
660+
661+
const modulePath = "m.com/a"
662+
663+
insert := func(version string, status int, modfile string) {
664+
m := sample.Module(modulePath, version, "pkg")
665+
lmv := addLatest(ctx, t, testDB, modulePath, version, "module "+modulePath+"\n"+modfile)
666+
m.Packages()[0].Documentation[0].Synopsis = version
667+
if status == 200 {
668+
MustInsertModuleLMV(ctx, t, testDB, m, lmv)
669+
}
670+
if err := testDB.UpsertModuleVersionState(ctx, &ModuleVersionStateForUpsert{
671+
ModulePath: modulePath,
672+
Version: version,
673+
AppVersion: "app",
674+
Timestamp: time.Now(),
675+
Status: status,
676+
}); err != nil {
677+
t.Fatal(err)
678+
}
679+
if err := testDB.ReInsertLatestVersion(ctx, modulePath); err != nil {
680+
t.Fatal(err)
681+
}
682+
}
683+
684+
check := func(wantVersion string) {
685+
var gotVersion, gotSynopsis string
686+
if err := testDB.db.QueryRow(ctx, `
687+
SELECT version, synopsis
688+
FROM search_documents
689+
WHERE module_path = 'm.com/a'
690+
AND package_path = 'm.com/a/pkg'
691+
`).Scan(&gotVersion, &gotSynopsis); err != nil {
692+
t.Fatal(err)
693+
}
694+
if gotVersion != wantVersion && gotSynopsis != wantVersion {
695+
t.Fatalf("got (%s, %s), want %s for both", gotVersion, gotSynopsis, wantVersion)
696+
}
697+
}
698+
699+
// Insert a good module. It should be in search_documents.
700+
insert("v1.1.0", 200, "")
701+
check("v1.1.0")
702+
703+
// Insert a higher, good version. It should replace the first.
704+
insert("v1.2.0", 200, "")
705+
check("v1.2.0")
706+
707+
// Now an even higher, bad version comes along that retracts v1.2.0.
708+
// The search_documents table should go back to v1.1.0.
709+
insert("v1.3.0", 400, "retract v1.2.0")
710+
check("v1.1.0")
711+
}

‎internal/postgres/search.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@ var upsertSearchStatement = fmt.Sprintf(`
467467
END)
468468
;`, hllRegisterCount)
469469

470-
// upsertSearchDocuments adds search information for mod ot the search_documents table.
470+
// upsertSearchDocuments adds search information for mod to the search_documents table.
471471
// It assumes that all non-redistributable data has been removed from mod.
472472
func upsertSearchDocuments(ctx context.Context, ddb *database.DB, mod *internal.Module) (err error) {
473473
defer derrors.WrapStack(&err, "upsertSearchDocuments(ctx, %q, %q)", mod.ModulePath, mod.Version)
@@ -513,6 +513,7 @@ func UpsertSearchDocument(ctx context.Context, ddb *database.DB, args UpsertSear
513513
defer derrors.WrapStack(&err, "DB.UpsertSearchDocument(ctx, ddb, %q, %q)", args.PackagePath, args.ModulePath)
514514

515515
// Only summarize the README if the package and module have the same path.
516+
// If this changes, fix DB.ReInsertLatestVersion.
516517
if args.PackagePath != args.ModulePath {
517518
args.ReadmeFilePath = ""
518519
args.ReadmeContents = ""

‎internal/postgres/unit.go

+13-5
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,11 @@ func (db *DB) getImports(ctx context.Context, unitID int) (_ []string, err error
448448
// getPackagesInUnit returns all of the packages in a unit from a
449449
// module version, including the package that lives at fullPath, if present.
450450
func (db *DB) getPackagesInUnit(ctx context.Context, fullPath, modulePath, resolvedVersion string) (_ []*internal.PackageMeta, err error) {
451-
defer derrors.WrapStack(&err, "DB.getPackagesInUnit(ctx, %q, %q, %q)", fullPath, modulePath, resolvedVersion)
451+
return getPackagesInUnit(ctx, db.db, fullPath, modulePath, resolvedVersion, db.bypassLicenseCheck)
452+
}
453+
454+
func getPackagesInUnit(ctx context.Context, db *database.DB, fullPath, modulePath, resolvedVersion string, bypassLicenseCheck bool) (_ []*internal.PackageMeta, err error) {
455+
defer derrors.WrapStack(&err, "getPackagesInUnit(ctx, %q, %q, %q)", fullPath, modulePath, resolvedVersion)
452456
defer middleware.ElapsedStat(ctx, "getPackagesInUnit")()
453457

454458
query := `
@@ -512,7 +516,7 @@ func (db *DB) getPackagesInUnit(ctx context.Context, fullPath, modulePath, resol
512516
}
513517
return nil
514518
}
515-
if err := db.db.RunQuery(ctx, query, collect, modulePath, resolvedVersion); err != nil {
519+
if err := db.RunQuery(ctx, query, collect, modulePath, resolvedVersion); err != nil {
516520
return nil, err
517521
}
518522

@@ -523,7 +527,7 @@ func (db *DB) getPackagesInUnit(ctx context.Context, fullPath, modulePath, resol
523527
}
524528
sort.Slice(packages, func(i, j int) bool { return packages[i].Path < packages[j].Path })
525529
for _, p := range packages {
526-
if db.bypassLicenseCheck {
530+
if bypassLicenseCheck {
527531
p.IsRedistributable = true
528532
} else {
529533
p.RemoveNonRedistributableData()
@@ -678,9 +682,13 @@ func (db *DB) getPathsInModule(ctx context.Context, modulePath, resolvedVersion
678682

679683
// GetModuleReadme returns the README corresponding to the modulePath and version.
680684
func (db *DB) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (_ *internal.Readme, err error) {
681-
defer derrors.WrapStack(&err, "GetModuleReadme(ctx, %q, %q)", modulePath, resolvedVersion)
685+
return getModuleReadme(ctx, db.db, modulePath, resolvedVersion)
686+
}
687+
688+
func getModuleReadme(ctx context.Context, db *database.DB, modulePath, resolvedVersion string) (_ *internal.Readme, err error) {
689+
defer derrors.WrapStack(&err, "getModuleReadme(ctx, %q, %q)", modulePath, resolvedVersion)
682690
var readme internal.Readme
683-
err = db.db.QueryRow(ctx, `
691+
err = db.QueryRow(ctx, `
684692
SELECT file_path, contents
685693
FROM modules m
686694
INNER JOIN units u

0 commit comments

Comments
 (0)