Add support for include_dir in config file.

This allows easily splitting configuration into many files, deployed in a
directory.

Magnus Hagander, Greg Smith, Selena Deckelmann, reviewed by Noah Misch.

3 files changed, 173 insertions, 29 deletions

diff --git a/src/backend/utils/misc/guc-file.l b/src/backend/utils/misc/guc-file.l
index ca7619034f4..52d540e4cd2 100644
--- a/src/backend/utils/misc/guc-file.l
+++ b/src/backend/utils/misc/guc-file.l
@@ -363,6 +363,39 @@ ProcessConfigFile(GucContext context)
 }
 
 /*
+ * Given a configuration file or directory location that may be a relative
+ * path, return an absolute one.  We consider the location to be relative to
+ * the directory holding the calling file.
+ */
+static char *
+AbsoluteConfigLocation(const char *location, const char *calling_file)
+{
+	char		abs_path[MAXPGPATH];
+
+	if (is_absolute_path(location))
+		return pstrdup(location);
+	else
+	{
+		if (calling_file != NULL)
+		{
+			strlcpy(abs_path, calling_file, sizeof(abs_path));
+			get_parent_directory(abs_path);
+			join_path_components(abs_path, abs_path, location);
+			canonicalize_path(abs_path);
+		}
+		else
+		{
+			/*
+			 * calling_file is NULL, we make an absolute path from $PGDATA
+			 */
+			join_path_components(abs_path, data_directory, location);
+			canonicalize_path(abs_path);
+		}
+		return pstrdup(abs_path);
+	}
+}
+
+/*
  * Read and parse a single configuration file.  This function recurses
  * to handle "include" directives.
  *
@@ -378,7 +411,6 @@ ParseConfigFile(const char *config_file, const char *calling_file, bool strict,
 {
 	bool		OK = true;
 	FILE	   *fp;
-	char		abs_path[MAXPGPATH];
 
 	/*
 	 * Reject too-deep include nesting depth.  This is just a safety check
@@ -394,31 +426,7 @@ ParseConfigFile(const char *config_file, const char *calling_file, bool strict,
 		return false;
 	}
 
-	/*
-	 * If config_file is a relative path, convert to absolute.  We consider
-	 * it to be relative to the directory holding the calling file.
-	 */
-	if (!is_absolute_path(config_file))
-	{
-		if (calling_file != NULL)
-		{
-			strlcpy(abs_path, calling_file, sizeof(abs_path));
-			get_parent_directory(abs_path);
-			join_path_components(abs_path, abs_path, config_file);
-			canonicalize_path(abs_path);
-			config_file = abs_path;
-		}
-		else
-		{
-			/*
-			 * calling_file is NULL, we make an absolute path from $PGDATA
-			 */
-			join_path_components(abs_path, data_directory, config_file);
-			canonicalize_path(abs_path);
-			config_file = abs_path;
-		}
-	}
-
+	config_file = AbsoluteConfigLocation(config_file,calling_file);
 	fp = AllocateFile(config_file, "r");
 	if (!fp)
 	{
@@ -563,20 +571,35 @@ ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,
 		}
 
 		/* OK, process the option name and value */
-		if (guc_name_compare(opt_name, "include_if_exists") == 0)
+		if (guc_name_compare(opt_name, "include_dir") == 0)
 		{
 			/*
-			 * An include_if_exists directive isn't a variable and should be
+			 * An include_dir directive isn't a variable and should be
 			 * processed immediately.
 			 */
-			if (!ParseConfigFile(opt_value, config_file, false,
+			if (!ParseConfigDirectory(opt_value, config_file,
 								 depth + 1, elevel,
 								 head_p, tail_p))
 				OK = false;
 			yy_switch_to_buffer(lex_buffer);
+			ConfigFileLineno = save_ConfigFileLineno;
 			pfree(opt_name);
 			pfree(opt_value);
 		}
+		else if (guc_name_compare(opt_name, "include_if_exists") == 0)
+		{
+			/*
+			 * An include_if_exists directive isn't a variable and should be
+			 * processed immediately.
+			 */
+			if (!ParseConfigFile(opt_value, config_file, false,
+								 depth + 1, elevel,
+								 head_p, tail_p))
+				OK = false;
+				yy_switch_to_buffer(lex_buffer);
+				pfree(opt_name);
+				pfree(opt_value);
+		}
 		else if (guc_name_compare(opt_name, "include") == 0)
 		{
 			/*
@@ -665,6 +688,111 @@ cleanup:
 	return OK;
 }
 
+/*
+ * Read and parse all config files in a subdirectory in alphabetical order
+ */
+bool
+ParseConfigDirectory(const char *includedir,
+					 const char *calling_file,
+					 int depth, int elevel,
+					 ConfigVariable **head_p,
+					 ConfigVariable **tail_p)
+{
+	char	   *directory;
+	DIR		   *d;
+	struct dirent *de;
+	char	  **filenames = NULL;
+	int			num_filenames = 0;
+	int			size_filenames = 0;
+	bool		status;
+
+	directory = AbsoluteConfigLocation(includedir, calling_file);
+	d = AllocateDir(directory);
+	if (d == NULL)
+	{
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not open configuration directory \"%s\": %m",
+						directory)));
+		return false;
+	}
+
+	/*
+	 * Read the directory and put the filenames in an array, so we can sort
+	 * them prior to processing the contents.
+	 */
+	while ((de = ReadDir(d, directory)) != NULL)
+	{
+		struct stat st;
+		char	filename[MAXPGPATH];
+
+		/*
+		 * Only parse files with names ending in ".conf".  Explicitly reject
+		 * files starting with ".".  This excludes things like "." and "..",
+		 * as well as typical hidden files, backup files, and editor debris.
+		 */
+		if (strlen(de->d_name) < 6)
+			continue;
+		if (de->d_name[0] == '.')
+			continue;
+		if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0)
+			continue;
+
+		join_path_components(filename, directory, de->d_name);
+		canonicalize_path(filename);
+		if (stat(filename, &st) == 0)
+		{
+			if (!S_ISDIR(st.st_mode))
+			{
+				/* Add file to list, increasing its size in blocks of 32 */
+				if (num_filenames == size_filenames)
+				{
+					size_filenames += 32;
+					if (num_filenames == 0)
+						/* Must initialize, repalloc won't take NULL input */
+						filenames = palloc(size_filenames * sizeof(char *));
+					else
+						filenames = repalloc(filenames, size_filenames * sizeof(char *));
+				}
+				filenames[num_filenames] = pstrdup(filename);
+				num_filenames++;
+			}
+		}
+		else
+		{
+			/*
+			 * stat does not care about permissions, so the most likely reason
+			 * a file can't be accessed now is if it was removed between the
+			 * directory listing and now.
+			 */
+			ereport(elevel,
+					(errcode_for_file_access(),
+					 errmsg("could not stat file \"%s\": %m",
+							filename)));
+			return false;
+		}
+	}
+
+	if (num_filenames > 0)
+	{
+		int			i;
+		qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp);
+		for (i = 0; i < num_filenames; i++)
+		{
+			if (!ParseConfigFile(filenames[i], NULL, true,
+								 depth, elevel, head_p, tail_p))
+			{
+				status = false;
+				goto cleanup;
+			}
+		}
+	}
+	status = true;
+
+cleanup:
+	FreeDir(d);
+	return status;
+}
 
 /*
  * Free a list of ConfigVariables, including the names and the values
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index adcbcf66205..10f3fb1b247 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -566,6 +566,17 @@
 #exit_on_error = off			# terminate session on any error?
 #restart_after_crash = on		# reinitialize after backend crash?
 
+#------------------------------------------------------------------------------
+# CONFIG FILE INCLUDES
+#------------------------------------------------------------------------------
+
+# These options allow settings to be loaded from files other than the
+# default postgresql.conf
+
+#include_dir = 'conf.d'			# include files ending in '.conf' from
+					# directory 'conf.d'
+#include_if_exists = 'exists.conf'	# include file only if it exists
+#include = 'special.conf'		# include file
 
 #------------------------------------------------------------------------------
 # CUSTOMIZED OPTIONS
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 68103877554..06f797cb0af 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -116,6 +116,11 @@ extern bool ParseConfigFile(const char *config_file, const char *calling_file,
 extern bool ParseConfigFp(FILE *fp, const char *config_file,
 			  int depth, int elevel,
 			  ConfigVariable **head_p, ConfigVariable **tail_p);
+extern bool ParseConfigDirectory(const char *includedir,
+					 const char *calling_file,
+					 int depth, int elevel,
+					 ConfigVariable **head_p,
+					 ConfigVariable **tail_p);
 extern void FreeConfigVariables(ConfigVariable *list);
 
 /*