Provide a build-time option to store large relations as single files, rather

than dividing them into 1GB segments as has been our longtime practice.  This
requires working support for large files in the operating system; at least for
the time being, it won't be the default.

Zdenek Kotala

2 files changed, 49 insertions, 45 deletions

diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c
index 8d79e9574b8..94e5c67911a 100644
--- a/src/backend/storage/file/buffile.c
+++ b/src/backend/storage/file/buffile.c
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/file/buffile.c,v 1.29 2008/01/01 19:45:51 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/file/buffile.c,v 1.30 2008/03/10 20:06:27 tgl Exp $
  *
  * NOTES:
  *
@@ -38,13 +38,12 @@
 #include "storage/buffile.h"
 
 /*
- * The maximum safe file size is presumed to be RELSEG_SIZE * BLCKSZ.
- * Note we adhere to this limit whether or not LET_OS_MANAGE_FILESIZE
- * is defined, although md.c ignores it when that symbol is defined.
- * The reason for doing this is that we'd like large temporary BufFiles
- * to be spread across multiple tablespaces when available.
+ * We break BufFiles into gigabyte-sized segments, whether or not
+ * USE_SEGMENTED_FILES is defined.  The reason is that we'd like large
+ * temporary BufFiles to be spread across multiple tablespaces when available.
  */
-#define MAX_PHYSICAL_FILESIZE  (RELSEG_SIZE * BLCKSZ)
+#define MAX_PHYSICAL_FILESIZE	0x40000000
+#define BUFFILE_SEG_SIZE		(MAX_PHYSICAL_FILESIZE / BLCKSZ)
 
 /*
  * This data structure represents a buffered file that consists of one or
@@ -56,7 +55,7 @@ struct BufFile
 	int			numFiles;		/* number of physical files in set */
 	/* all files except the last have length exactly MAX_PHYSICAL_FILESIZE */
 	File	   *files;			/* palloc'd array with numFiles entries */
-	long	   *offsets;		/* palloc'd array with numFiles entries */
+	off_t	   *offsets;		/* palloc'd array with numFiles entries */
 
 	/*
 	 * offsets[i] is the current seek position of files[i].  We use this to
@@ -72,7 +71,7 @@ struct BufFile
 	 * Position as seen by user of BufFile is (curFile, curOffset + pos).
 	 */
 	int			curFile;		/* file index (0..n) part of current pos */
-	int			curOffset;		/* offset part of current pos */
+	off_t		curOffset;		/* offset part of current pos */
 	int			pos;			/* next read/write position in buffer */
 	int			nbytes;			/* total # of valid bytes in buffer */
 	char		buffer[BLCKSZ];
@@ -97,7 +96,7 @@ makeBufFile(File firstfile)
 	file->numFiles = 1;
 	file->files = (File *) palloc(sizeof(File));
 	file->files[0] = firstfile;
-	file->offsets = (long *) palloc(sizeof(long));
+	file->offsets = (off_t *) palloc(sizeof(off_t));
 	file->offsets[0] = 0L;
 	file->isTemp = false;
 	file->isInterXact = false;
@@ -124,8 +123,8 @@ extendBufFile(BufFile *file)
 
 	file->files = (File *) repalloc(file->files,
 									(file->numFiles + 1) * sizeof(File));
-	file->offsets = (long *) repalloc(file->offsets,
-									  (file->numFiles + 1) * sizeof(long));
+	file->offsets = (off_t *) repalloc(file->offsets,
+									  (file->numFiles + 1) * sizeof(off_t));
 	file->files[file->numFiles] = pfile;
 	file->offsets[file->numFiles] = 0L;
 	file->numFiles++;
@@ -279,9 +278,9 @@ BufFileDumpBuffer(BufFile *file)
 		bytestowrite = file->nbytes - wpos;
 		if (file->isTemp)
 		{
-			long		availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
+			off_t		availbytes = MAX_PHYSICAL_FILESIZE - file->curOffset;
 
-			if ((long) bytestowrite > availbytes)
+			if ((off_t) bytestowrite > availbytes)
 				bytestowrite = (int) availbytes;
 		}
 
@@ -451,10 +450,10 @@ BufFileFlush(BufFile *file)
  * impossible seek is attempted.
  */
 int
-BufFileSeek(BufFile *file, int fileno, long offset, int whence)
+BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
 {
 	int			newFile;
-	long		newOffset;
+	off_t		newOffset;
 
 	switch (whence)
 	{
@@ -469,7 +468,7 @@ BufFileSeek(BufFile *file, int fileno, long offset, int whence)
 			/*
 			 * Relative seek considers only the signed offset, ignoring
 			 * fileno. Note that large offsets (> 1 gig) risk overflow in this
-			 * add...
+			 * add, unless we have 64-bit off_t.
 			 */
 			newFile = file->curFile;
 			newOffset = (file->curOffset + file->pos) + offset;
@@ -537,7 +536,7 @@ BufFileSeek(BufFile *file, int fileno, long offset, int whence)
 }
 
 void
-BufFileTell(BufFile *file, int *fileno, long *offset)
+BufFileTell(BufFile *file, int *fileno, off_t *offset)
 {
 	*fileno = file->curFile;
 	*offset = file->curOffset + file->pos;
@@ -558,8 +557,8 @@ int
 BufFileSeekBlock(BufFile *file, long blknum)
 {
 	return BufFileSeek(file,
-					   (int) (blknum / RELSEG_SIZE),
-					   (blknum % RELSEG_SIZE) * BLCKSZ,
+					   (int) (blknum / BUFFILE_SEG_SIZE),
+					   (off_t) (blknum % BUFFILE_SEG_SIZE) * BLCKSZ,
 					   SEEK_SET);
 }
 
@@ -575,7 +574,7 @@ BufFileTellBlock(BufFile *file)
 	long		blknum;
 
 	blknum = (file->curOffset + file->pos) / BLCKSZ;
-	blknum += file->curFile * RELSEG_SIZE;
+	blknum += file->curFile * BUFFILE_SEG_SIZE;
 	return blknum;
 }
 
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index 2a0108fcee0..edce52155f6 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/file/fd.c,v 1.143 2008/01/01 19:45:51 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/file/fd.c,v 1.144 2008/03/10 20:06:27 tgl Exp $
  *
  * NOTES:
  *
@@ -115,7 +115,7 @@ static int	max_safe_fds = 32;	/* default if not changed */
 
 #define FileIsNotOpen(file) (VfdCache[file].fd == VFD_CLOSED)
 
-#define FileUnknownPos (-1L)
+#define FileUnknownPos ((off_t) -1)
 
 /* these are the assigned bits in fdstate below: */
 #define FD_TEMPORARY		(1 << 0)	/* T = delete when closed */
@@ -123,13 +123,13 @@ static int	max_safe_fds = 32;	/* default if not changed */
 
 typedef struct vfd
 {
-	signed short fd;			/* current FD, or VFD_CLOSED if none */
+	int			fd;				/* current FD, or VFD_CLOSED if none */
 	unsigned short fdstate;		/* bitflags for VFD's state */
-	SubTransactionId create_subid;		/* for TEMPORARY fds, creating subxact */
+	SubTransactionId create_subid;	/* for TEMPORARY fds, creating subxact */
 	File		nextFree;		/* link to next free VFD, if in freelist */
 	File		lruMoreRecently;	/* doubly linked recency-of-use list */
 	File		lruLessRecently;
-	long		seekPos;		/* current logical file position */
+	off_t		seekPos;		/* current logical file position */
 	char	   *fileName;		/* name of file, or NULL for unused VFD */
 	/* NB: fileName is malloc'd, and must be free'd when closing the VFD */
 	int			fileFlags;		/* open(2) flags for (re)opening the file */
@@ -544,8 +544,8 @@ LruDelete(File file)
 	Delete(file);
 
 	/* save the seek position */
-	vfdP->seekPos = (long) lseek(vfdP->fd, 0L, SEEK_CUR);
-	Assert(vfdP->seekPos != -1L);
+	vfdP->seekPos = lseek(vfdP->fd, (off_t) 0, SEEK_CUR);
+	Assert(vfdP->seekPos != (off_t) -1);
 
 	/* close the file */
 	if (close(vfdP->fd))
@@ -616,12 +616,12 @@ LruInsert(File file)
 		}
 
 		/* seek to the right position */
-		if (vfdP->seekPos != 0L)
+		if (vfdP->seekPos != (off_t) 0)
 		{
-			long		returnValue;
+			off_t		returnValue;
 
-			returnValue = (long) lseek(vfdP->fd, vfdP->seekPos, SEEK_SET);
-			Assert(returnValue != -1L);
+			returnValue = lseek(vfdP->fd, vfdP->seekPos, SEEK_SET);
+			Assert(returnValue != (off_t) -1);
 		}
 	}
 
@@ -1027,9 +1027,10 @@ FileRead(File file, char *buffer, int amount)
 
 	Assert(FileIsValid(file));
 
-	DO_DB(elog(LOG, "FileRead: %d (%s) %ld %d %p",
+	DO_DB(elog(LOG, "FileRead: %d (%s) " INT64_FORMAT " %d %p",
 			   file, VfdCache[file].fileName,
-			   VfdCache[file].seekPos, amount, buffer));
+			   (int64) VfdCache[file].seekPos,
+			   amount, buffer));
 
 	returnCode = FileAccess(file);
 	if (returnCode < 0)
@@ -1081,9 +1082,10 @@ FileWrite(File file, char *buffer, int amount)
 
 	Assert(FileIsValid(file));
 
-	DO_DB(elog(LOG, "FileWrite: %d (%s) %ld %d %p",
+	DO_DB(elog(LOG, "FileWrite: %d (%s) " INT64_FORMAT " %d %p",
 			   file, VfdCache[file].fileName,
-			   VfdCache[file].seekPos, amount, buffer));
+			   (int64) VfdCache[file].seekPos,
+			   amount, buffer));
 
 	returnCode = FileAccess(file);
 	if (returnCode < 0)
@@ -1146,16 +1148,17 @@ FileSync(File file)
 	return pg_fsync(VfdCache[file].fd);
 }
 
-long
-FileSeek(File file, long offset, int whence)
+off_t
+FileSeek(File file, off_t offset, int whence)
 {
 	int			returnCode;
 
 	Assert(FileIsValid(file));
 
-	DO_DB(elog(LOG, "FileSeek: %d (%s) %ld %ld %d",
+	DO_DB(elog(LOG, "FileSeek: %d (%s) " INT64_FORMAT " " INT64_FORMAT " %d",
 			   file, VfdCache[file].fileName,
-			   VfdCache[file].seekPos, offset, whence));
+			   (int64) VfdCache[file].seekPos,
+			   (int64) offset, whence));
 
 	if (FileIsNotOpen(file))
 	{
@@ -1163,7 +1166,8 @@ FileSeek(File file, long offset, int whence)
 		{
 			case SEEK_SET:
 				if (offset < 0)
-					elog(ERROR, "invalid seek offset: %ld", offset);
+					elog(ERROR, "invalid seek offset: " INT64_FORMAT,
+						 (int64) offset);
 				VfdCache[file].seekPos = offset;
 				break;
 			case SEEK_CUR:
@@ -1187,7 +1191,8 @@ FileSeek(File file, long offset, int whence)
 		{
 			case SEEK_SET:
 				if (offset < 0)
-					elog(ERROR, "invalid seek offset: %ld", offset);
+					elog(ERROR, "invalid seek offset: " INT64_FORMAT,
+						 (int64) offset);
 				if (VfdCache[file].seekPos != offset)
 					VfdCache[file].seekPos = lseek(VfdCache[file].fd,
 												   offset, whence);
@@ -1213,7 +1218,7 @@ FileSeek(File file, long offset, int whence)
  * XXX not actually used but here for completeness
  */
 #ifdef NOT_USED
-long
+off_t
 FileTell(File file)
 {
 	Assert(FileIsValid(file));
@@ -1224,7 +1229,7 @@ FileTell(File file)
 #endif
 
 int
-FileTruncate(File file, long offset)
+FileTruncate(File file, off_t offset)
 {
 	int			returnCode;
 
@@ -1237,7 +1242,7 @@ FileTruncate(File file, long offset)
 	if (returnCode < 0)
 		return returnCode;
 
-	returnCode = ftruncate(VfdCache[file].fd, (size_t) offset);
+	returnCode = ftruncate(VfdCache[file].fd, offset);
 	return returnCode;
 }