1. Red Hat People
2. Daniel Berrange
3. Large File Support

Large File Support (LFS) on RHEL 3

On 32-bit architectures of RHEL 3, the maximum filesize that can be handled by a program is traditionally 2 GB (2^31 - 1 bytes). Many filesystems in Linux support creation of files larger than this limit. For example, in the RHEL 3 kernel, ext2/3 allow files upto 1 TB in size, with a total filesystem size of 8TB. To go beyond the 2 GB barrier and reach the underlying filesystem limits a program must be made aware of the Large File Support (LFS) standard.

To check if a filesystem supports the LFS standard, the getconf program should be used

  $ getconf FILESIZEBITS /some/path

...where /some/path is the name of a file or directory on the filesystems to check. If the result is 64, LFS is supported.

Compiling with LFS

There are two approaches to adding LFS support to an application, which we'll classify as implicit and explicit. With implicit support, pre-processor options are used to transparently switch all filesize related data types to 64-bits in size & alias all functions to their 64-bit variants. With explicit support, pre-processor options are used to make 64-bit data types & functions available for use by the programmer. (In the context of the LFS standard, explicit support is detailed under the heading Transitional Extensions to the Single UNIX Specification.) Explicit support is the preferred approach for any shared libraries, or programs that will need to link against libraries other than GLibC.

Implicit LFS

The compiler is invoked with the additional options:

  -D_LARGEFILE_SOURCE ... $(getconf LFS_CFLAGS)

The linker is invoked with the additional otions:

  $(getconf LFS_LDFLAGS) ... $(getconf LFS_LIBS)

These options have three effects:

1. Enable visibility of the fseeko and ftello functions
2. Calls to functions dealing with files are redirected to their 64-bit variants. eg open -> open64, stat -> stat64
3. The size of filesize related data types is increased to 64 bits. eg off_t, blkcnt_t, etc are now 64 bits in size

While this method does have the advantage that no source code changes are required, an important caveat is that great care must be taken not to link against 3rd party libraries compiled without/with differing LFS support

Explicit LFS

The compiler is invoked with the additional options:

  -D_LARGEFILE64_SOURCE ... $(getconf LFS64_CFLAGS)

The linker is invoked with the additional options:

  $(getconf LFS64_LDFLAGS) ... $(getconf LFS64_LIBS)

These options have three effects:

1. Enable visibility of the fseeko and ftello functions
2. Enable visibility of the 64-bit variants of all functions dealing with files. eg open64, stat64, etc
3. Enable visibility of the 64-bit variants of all data types dealing with file sizes. eg off64_t, blkcnt64_t

Since the existing 32-bit functions and data types are not changed, programs will have to be changed to explicitly use the 64-bit variants when available. This does, however, provide greater reliabilty when linking against 3rd party libraries.

Related resources

For complete details of the three pre-processor macros listed above, read the GLibC info pages:

  info libc 'Feature Test Macros'

APIs with 64-bit variants

With explicit LFS support (ie -DLARGEFILE64_SOURCE), the following additional data types, structures, constants and functions are available. This is a summary of information provided in the LFS standard, and the GLibC info pages.

<aio.h>

  struct aiocb64

  int     aio_read64(struct aiocb64 *aiocbp);
  int     aio_write64(struct aiocb64 *aiocbp);
  int     lio_listio64(int mode,
                       struct aiocb64 *const list,
                       int nent, struct sigevent *restrict sig);
  int     aio_error64(const struct aiocb64 *aiocbp);
  ssize_t aio_return64(struct aiocb64 *aiocbp);
  int     aio_cancel64(int fildes, struct aiocb64 *aiocbp);
  int     aio_suspend64(const struct aiocb64 *const list[], int nent,
                        const struct timespec *restrict timeout);
  int     aio_fsync64(int operation, struct aiocb64 *aiocbp);

<dirent.h>

  struct dirent64;

  struct dirent64 *readdir64(DIR *dirp);

<fcntl.h>

  F_GETLK64
  F_SETLK64
  F_SETLKW64

  struct flock64;

  int creat64(const char *path, mode_t mode);
  int open64(const char *path, int oflag, ...);

<ftw.h>

  int ftw64(const char *path,
            int (*fn)(const char *, const struct stat64 *, int),
            int ndirs);
  int nftw64(const char *path,
             int (*fn)(const char *, const struct stat64 *, int,
                       struct FTW *),
             int depth, int flags);

<glob.h>

  glob64_t

  int  glob64(const char *pattern, int __flags,
              int (*errfunc) (const char *, int),
              glob64_t *pglob);

  void globfree64(glob64_t *pglob);

<stdio.h>

  fpos64_t;

  int       fgetpos64(FILE *stream, fpos64_t *pos);
  FILE     *fopen64(const char *filename, const char *mode);
  FILE     *freopen64(const char *filename, const char *mode,
                      FILE *stream);
  int       fseeko64(FILE *stream, off64_t offset, int whence);
  int       fsetpos64(FILE *stream, const fpos64_t *pos);
  off64_t   ftello64(FILE *stream);
  FILE     *tmpfile64(void);

<stdlib.h>

  int mkstemp64 (char *template);

<sys/mman.h>

  void     *mmap64(void *addr, size_t len, int prot, int flags,
                   int fd, off64_t offset);

<sys/resource.h>

  RLIM64_INFINITY
  RLIM64_SAVED_MAX
  RLIM64_SAVED_CUR

  rlim64_t;

  struct rlimit64;

  int       getrlimit64(int resource, struct rlimit64 *rlp);
  int       setrlimit64(int resource, const struct rlimit64 *rlp);

<sys/sendfile.h>

  ssize_t sendfile64(int out_fd, int in_fd, off64_t *offset,
                     size_t count);

<sys/stat.h>

  struct stat64;

  int         fstat64(int fildes, struct stat64 *buf);
  int         lstat64(const char *, struct stat64 *buf);
  int         stat64(const char *, struct stat64 *buf);

<sys/statfs.h>

  struct statfs64;

  int fstatfs64 (int fildes, struct statfs64 *buf);

<sys/statvfs.h>

  struct statvfs64;

  int         statvfs64(const char *path, struct statvfs64 *buf);
  int         fstatvfs64(int fildes, struct statvfs64 *buf);

<sys/types.h>

   blkcnt64_t
   fsblkcnt64_t
   fsfilcnt64_t
   ino64_t
   off64_t

<unistd.h>

  int         lockf64(int fildes, int function, off64_t size);
  off64_t     lseek64(int fildes, off64_t offset, int whence);
  int         ftruncate64(int fildes, off64_t length);
  int         truncate64(const char *path, off64_t length);

Last updated on Thursday, Sep 23, 2004.