Lockdir prefix for install packages

doc/NEWS.Rd

@@ -26,7 +26,15 @@
 
   \subsection{PACKAGE INSTALLATION}{
     \itemize{
-      \item .
+
+      \item \code{install.packages()} supports setting a different path for the
+	  intermediate \code{00LOCK} directories through the environment variable
+	  \code{R_LOCKDIR_PREFIX}. This helps source package installation in some
+	  distributed filesystems available in cloud environments like databricks.
+	  Check the
+	  \dQuote{R Installation and Administration} manual for further details on
+	  this option.
+
     }
   }
 
@@ -423,6 +431,7 @@
 
       \item There is preliminary support for C++26 with GCC \eqn{>=} 14, Apple
       \command{clang++} \eqn{>=} 16 and \I{LLVM} \command{clang++} \eqn{>=} 17.
+
     }
   }
 

doc/manual/R-admin.texi

@@ -2061,6 +2061,20 @@ comparable age to your version of @R{}).
 Naive users sometimes forget that as well as installing a package, they
 have to use @code{library} to make its functionality available.
 
+The package installation process has some standard requirements on the
+file system where packages are installed. Specifically, when a \emph{source
+package} needs to be installed, R needs to be able
+to open files in append mode \code{open(filename, "a")}. R also expects to
+be able to perform atomic directory moves during package installation
+to ensure correct error recovery from package installation errors (see
+@uref{https://developer.r-project.org/Blog/public/2019/02/14/staged-install/index.html}).
+Both requirements are fullfilled by POSIX and Windows filesystems, but not by
+some distributed file systems. In scenarios where opening append mode is not available, the
+@env{R_LOCKDIR_PREFIX} environment variable can be set to specify the absolute path to
+a different directory (in a compliant file system) where the package can be preinstalled
+before being moved to its final destination. While this approach makes package installation
+in those file systems possible, the error recovery capabilities will not be as robust as
+on supported filesystems.
 
 @node Windows packages
 @subsection Windows

src/library/tools/R/install.R

@@ -54,7 +54,23 @@ if(FALSE) {
 
     ## global variables
     curPkg <- character() # list of packages in current pkg
+
+    ## Path to where lockdirs will be created. If empty, use lib directory
+    lockdir_prefix <- Sys.getenv("R_LOCKDIR_PREFIX", "")
+    if (lockdir_prefix == "")
+        lockdir_prefix <- NULL
+    else
+        lockdir_prefix <- path.expand(lockdir_prefix)
+    ## The lockdir, relative to lockdir_prefix
     lockdir <- ""
+    ## The lockdir, including the prefix
+    lockdir_with_prefix <- ""
+    get_lockdir_with_prefix <- function(lockdir) {
+        if (!is.null(lockdir_prefix))
+            file.path(lockdir_prefix, lockdir)
+        else
+            lockdir
+    }
     is_first_package <- TRUE
     stars <- "*"
     user.tmpdir <- Sys.getenv("PKG_BUILD_DIR")
@@ -105,8 +121,8 @@ if(FALSE) {
             }
 
             if (nzchar(lockdir) &&
-                dir.exists(lp <- file.path(lockdir, curPkg)) &&
-                is_subdir(lp, lockdir)) {
+                dir.exists(lp <- file.path(lockdir_with_prefix, curPkg)) &&
+                is_subdir(lp, lockdir_with_prefix)) {
                 starsmsg(stars, "restoring previous ", sQuote(pkgdir))
                 if (WINDOWS) {
                     file.copy(lp, dirname(pkgdir), recursive = TRUE,
@@ -134,7 +150,7 @@ if(FALSE) {
             if (lib == .Library && "html" %in% build_help_types)
                 utils::make.packages.html(.Library, docdir = R.home("doc"))
         }
-        if (nzchar(lockdir)) unlink(lockdir, recursive = TRUE)
+        if (nzchar(lockdir_with_prefix)) unlink(lockdir_with_prefix, recursive = TRUE)
     }
 
     do_cleanup_tmpdir <- function()
@@ -478,7 +494,7 @@ if(FALSE) {
         if (file.exists(file.path(instdir, "DESCRIPTION"))) {
             if (nzchar(lockdir))
                 system(paste("mv -f", shQuote(instdir),
-                             shQuote(file.path(lockdir, pkg))))
+                             shQuote(file.path(lockdir_with_prefix, pkg))))
             dir.create(instdir, recursive = TRUE, showWarnings = FALSE)
         }
         TAR <- Sys.getenv("TAR", 'tar')
@@ -1041,9 +1057,9 @@ if(FALSE) {
                     if (more_than_libs) unlink(instdir, recursive = TRUE)
                 } else if (more_than_libs)
                     system(paste("mv -f ", shQuote(instdir),
-                                 shQuote(file.path(lockdir, pkg_name))))
+                                 shQuote(file.path(lockdir_with_prefix, pkg_name))))
                 else
-                    file.copy(instdir, lockdir, recursive = TRUE,
+                    file.copy(instdir, lockdir_with_prefix, recursive = TRUE,
                               copy.date = TRUE)
             } else if (more_than_libs) unlink(instdir, recursive = TRUE)
             if (more_than_libs && dir.exists(instdir))
@@ -1073,10 +1089,10 @@ if(FALSE) {
             final_rlibs <- Sys.getenv("R_LIBS")
             final_libpaths <- .libPaths()
 
-            instdir <- file.path(lockdir, "00new", pkg_name)
+            instdir <- file.path(lockdir_with_prefix, "00new", pkg_name)
             Sys.setenv(R_PACKAGE_DIR = instdir)
             dir.create(instdir, recursive = TRUE, showWarnings = FALSE)
-            lib <- file.path(lockdir, "00new")
+            lib <- file.path(lockdir_with_prefix, "00new")
 
             rlibs <- if (nzchar(final_rlibs))
                          paste(lib, final_rlibs, sep = .Platform$path.sep)
@@ -2373,7 +2389,8 @@ if(FALSE) {
     }
     if (lock && !pkglock) {
         lockdir <- file.path(lib, "00LOCK")
-        mk_lockdir(lockdir)
+        lockdir_with_prefix <- get_lockdir_with_prefix(lockdir)
+        mk_lockdir(lockdir_with_prefix)
     }
     if (is.na(staged_install)) {
         # environment variable intended as temporary
@@ -2418,7 +2435,8 @@ if(FALSE) {
     for(pkg in allpkgs) {
         if (pkglock) {
             lockdir <- file.path(lib, paste0("00LOCK-", basename(pkg)))
-            mk_lockdir(lockdir)
+            lockdir_with_prefix <- get_lockdir_with_prefix(lockdir)
+            mk_lockdir(lockdir_with_prefix)
         }
         do_install(pkg)
     }