Refresh when local file changes (#1489)

* Refresh when local file changes

* Fixing test failure

runtime/services/catalog/migrations.go

@@ -297,6 +297,8 @@ func (s *Service) getMigrationItem(
 		Path: repoPath,
 	}
 
+	forceChange := forcedPathMap[repoPath]
+
 	catalog, err := artifacts.Read(ctx, s.Repo, s.InstId, repoPath)
 	if err != nil {
 		if err != artifacts.FileReadError {
@@ -324,8 +326,15 @@ func (s *Service) getMigrationItem(
 			item.NormalizedDependencies[i] = strings.ToLower(dep)
 		}
 		repoStat, _ := s.Repo.Stat(ctx, s.InstId, repoPath)
-		item.CatalogInFile.UpdatedOn = repoStat.LastUpdated
-		if repoStat.LastUpdated.After(s.LastMigration) {
+		catalogLastUpdated, _ := migrator.LastUpdated(ctx, s.InstId, s.Repo, catalog)
+		if repoStat.LastUpdated.After(catalogLastUpdated) {
+			item.CatalogInFile.UpdatedOn = repoStat.LastUpdated
+		} else {
+			item.CatalogInFile.UpdatedOn = catalogLastUpdated
+			// if catalog has changed in anyway then always re-create/update
+			forceChange = true
+		}
+		if item.CatalogInFile.UpdatedOn.After(s.LastMigration) {
 			// assume creation until we see a catalog object
 			item.Type = MigrationCreate
 		}
@@ -356,7 +365,7 @@ func (s *Service) getMigrationItem(
 
 	switch item.Type {
 	case MigrationCreate:
-		if migrator.IsEqual(ctx, item.CatalogInFile, item.CatalogInStore) && !forcedPathMap[repoPath] {
+		if migrator.IsEqual(ctx, item.CatalogInFile, item.CatalogInStore) && !forceChange {
 			// if the actual content has not changed, mark as MigrationNoChange
 			item.Type = MigrationNoChange
 		} else {
@@ -366,7 +375,7 @@ func (s *Service) getMigrationItem(
 
 	case MigrationNoChange:
 		// if item doesn't exist in olap, mark as create
-		// TODO: is this path ever hit?
+		// happens when the catalog table is modified directly
 		ok, _ := migrator.ExistsInOlap(ctx, s.Olap, item.CatalogInFile)
 		if !ok {
 			item.Type = MigrationCreate

runtime/services/catalog/migrations_test.go

@@ -15,6 +15,7 @@ import (
 	_ "github.com/rilldata/rill/runtime/drivers/file"
 	_ "github.com/rilldata/rill/runtime/drivers/sqlite"
 	"github.com/rilldata/rill/runtime/services/catalog"
+	"github.com/rilldata/rill/runtime/services/catalog/artifacts"
 	_ "github.com/rilldata/rill/runtime/services/catalog/artifacts/sql"
 	_ "github.com/rilldata/rill/runtime/services/catalog/artifacts/yaml"
 	"github.com/rilldata/rill/runtime/services/catalog/migrator/metrics_views"
@@ -192,14 +193,39 @@ func TestRefreshSource(t *testing.T) {
 
 	for _, tt := range configs {
 		t.Run(tt.title, func(t *testing.T) {
-			s, _ := initBasicService(t)
+			s, dir := initBasicService(t)
+
+			testutils.CopyFileToData(t, dir, AdBidsCsvPath)
+			AdBidsDataPath := "data/AdBids.csv"
 
 			// update with same content
-			testutils.CreateSource(t, s, "AdBids", AdBidsCsvPath, AdBidsRepoPath)
+			err := artifacts.Write(context.Background(), s.Repo, s.InstId, &drivers.CatalogEntry{
+				Name: "AdBids",
+				Type: drivers.ObjectTypeSource,
+				Path: AdBidsRepoPath,
+				Object: &runtimev1.Source{
+					Name:      "AdBids",
+					Connector: "local_file",
+					Properties: testutils.ToProtoStruct(map[string]any{
+						"path": AdBidsDataPath,
+					}),
+				},
+			})
+			require.NoError(t, err)
 			result, err := s.Reconcile(context.Background(), tt.config)
 			require.NoError(t, err)
 			// ForcedPaths updates all dependant items
 			testutils.AssertMigration(t, result, 0, 0, 3, 0, AdBidsAffectedPaths)
+
+			// update the uploaded file directly
+			time.Sleep(10 * time.Millisecond)
+			err = os.Chtimes(path.Join(dir, AdBidsDataPath), time.Now(), time.Now())
+			require.NoError(t, err)
+			result, err = s.Reconcile(context.Background(), catalog.ReconcileConfig{
+				ChangedPaths: tt.config.ChangedPaths,
+			})
+			require.NoError(t, err)
+			testutils.AssertMigration(t, result, 0, 0, 3, 0, AdBidsAffectedPaths)
 		})
 	}
 }

runtime/services/catalog/migrator/migrator.go

@@ -3,6 +3,7 @@ package migrator
 import (
 	"context"
 	"fmt"
+	"time"
 
 	runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
 	"github.com/rilldata/rill/runtime/drivers"
@@ -138,6 +139,19 @@ func SetSchema(ctx context.Context, olap drivers.OLAPStore, catalog *drivers.Cat
 	return nil
 }
 
+func LastUpdated(ctx context.Context, instID string, repo drivers.RepoStore, catalog *drivers.CatalogEntry) (time.Time, error) {
+	// TODO: do we need to push this to individual implementations to handle just local_file source?
+	if catalog.Type != drivers.ObjectTypeSource || catalog.GetSource().Connector != "local_file" {
+		// return a very old time
+		return time.Time{}, nil
+	}
+	stat, err := repo.Stat(ctx, instID, catalog.GetSource().Properties.Fields["path"].GetStringValue())
+	if err != nil {
+		return time.Time{}, err
+	}
+	return stat.LastUpdated, nil
+}
+
 func CreateValidationError(filePath string, message string) []*runtimev1.ReconcileError {
 	return []*runtimev1.ReconcileError{
 		{

runtime/services/catalog/testutils/testutils.go

@@ -3,6 +3,7 @@ package testutils
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -30,7 +31,7 @@ func CreateSource(t *testing.T, s *catalog.Service, name string, file string, pa
 		Object: &runtimev1.Source{
 			Name:      name,
 			Connector: "local_file",
-			Properties: toProtoStruct(map[string]any{
+			Properties: ToProtoStruct(map[string]any{
 				"path": absFile,
 			}),
 		},
@@ -75,7 +76,7 @@ func CreateMetricsView(t *testing.T, s *catalog.Service, metricsView *runtimev1.
 	return blob
 }
 
-func toProtoStruct(obj map[string]any) *structpb.Struct {
+func ToProtoStruct(obj map[string]any) *structpb.Struct {
 	s, err := structpb.NewStruct(obj)
 	if err != nil {
 		panic(err)
@@ -154,3 +155,22 @@ func RenameFile(t *testing.T, dir string, from string, to string) {
 	err = os.Chtimes(path.Join(dir, to), time.Now(), time.Now())
 	require.NoError(t, err)
 }
+
+func CopyFileToData(t *testing.T, dir string, source string) {
+	baseName := path.Base(source)
+	dest := path.Join(dir, "data", baseName)
+
+	err := os.MkdirAll(path.Join(dir, "data"), 0777)
+	require.NoError(t, err)
+
+	sourceFile, err := os.Open(source)
+	require.NoError(t, err)
+	defer sourceFile.Close()
+
+	destFile, err := os.Create(dest)
+	require.NoError(t, err)
+	defer destFile.Close()
+
+	_, err = io.Copy(destFile, sourceFile)
+	require.NoError(t, err)
+}

web-local/src/lib/components/navigation/sources/createSource.ts

@@ -23,7 +23,9 @@ export async function createSource(
       path: getFilePathFromNameAndType(tableName, EntityType.Table),
       blob: yaml,
       create: true,
-      createOnly: true,
+      // create source is used to upload and replace.
+      // so we cannot send createOnly=true until we refactor it to use refresh source
+      createOnly: false,
       strict: true,
     },
   });