From 61c0e893aad4ba476aa9e001a3df10844972b901 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 15 Nov 2011 09:36:13 -0800 Subject: [PATCH 1/5] CCLINK already includes ALLOC_LINK --- src/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Makefile b/src/Makefile index 6cb5f062..3ef18f07 100644 --- a/src/Makefile +++ b/src/Makefile @@ -174,26 +174,26 @@ dependencies: cd ../deps/jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a redis-server: dependencies $(OBJ) - $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) $(CCLINK) $(ALLOC_LINK) ../deps/lua/src/liblua.a + $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) $(CCLINK) ../deps/lua/src/liblua.a redis-benchmark: dependencies $(BENCHOBJ) @cd ../deps/hiredis && $(MAKE) static - $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) $(ALLOC_LINK) + $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) redis-benchmark.o: $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $< redis-cli: dependencies $(CLIOBJ) - $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) $(ALLOC_LINK) + $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) redis-cli.o: $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis -I../deps/linenoise $(DEBUG) $(COMPILE_TIME) $< redis-check-dump: $(CHECKDUMPOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) $(ALLOC_LINK) + $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) redis-check-aof: $(CHECKAOFOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) $(ALLOC_LINK) + $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) # Because the jemalloc.h header is generated as a part of the jemalloc build # process, building it should complete before building any other object. From 321a0440c0dde3ea00a72da0751fd7beb8f9bb93 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 15 Nov 2011 09:39:38 -0800 Subject: [PATCH 2/5] Use CCLINK as last argument --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 3ef18f07..76bc223b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -174,7 +174,7 @@ dependencies: cd ../deps/jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a redis-server: dependencies $(OBJ) - $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) $(CCLINK) ../deps/lua/src/liblua.a + $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) ../deps/lua/src/liblua.a $(CCLINK) redis-benchmark: dependencies $(BENCHOBJ) @cd ../deps/hiredis && $(MAKE) static From 4b8a63941dc360a0d2dcba1ec182a006221e4a20 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 15 Nov 2011 12:40:49 -0800 Subject: [PATCH 3/5] Rebuild deps/ and src/ when ARCH changes This change moves the build instructions for dependencies to a separate Makefile in deps/. The ARCH environment variable is stored in a .make-arch file in the same directory as the Makefile. The contents of this file is read and compared to the current ARCH, and, on a mismatch triggers rebuilding the entire source tree. When file .make-arch exists and matches with ARCH from the environment, the dependencies are assumed to already be built. The new "clean" target only cleans the Redis source tree, not its dependencies. To clear the dependencies as well, the "distclean" target can be used. --- .gitignore | 1 + deps/Makefile | 59 +++++++++++++++++++++++++++++++++++++++++++++++ src/Makefile | 63 +++++++++++++++++++++++---------------------------- 3 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 deps/Makefile diff --git a/.gitignore b/.gitignore index 262faef3..1f480bd4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ src/redis.conf deps/lua/src/lua deps/lua/src/luac deps/lua/src/liblua.a +.make-* diff --git a/deps/Makefile b/deps/Makefile new file mode 100644 index 00000000..b881c814 --- /dev/null +++ b/deps/Makefile @@ -0,0 +1,59 @@ +# Redis dependency Makefile + +UNAME_S:=$(shell sh -c 'uname -s 2> /dev/null || echo not') + +LUA_CFLAGS=-O2 -Wall $(ARCH) +ifeq ($(UNAME_S),SunOS) + # Make isinf() available + LUA_CFLAGS+= -D__C99FEATURES__=1 +endif + +JEMALLOC_CFLAGS= +ifeq ($(ARCH),-m32) + JEMALLOC_CFLAGS+=CFLAGS="-std=gnu99 -Wall -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -m32" +endif + +CCCOLOR="\033[34m" +LINKCOLOR="\033[34;1m" +SRCCOLOR="\033[33m" +BINCOLOR="\033[37;1m" +MAKECOLOR="\033[32;1m" +ENDCOLOR="\033[0m" + +default: + @echo "Explicit target required" + +# Clean everything when ARCH is different +ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) +.make-arch: distclean +else +.make-arch: +endif + +.make-arch: + -(echo $(ARCH) > .make-arch) + +distclean: + -(cd hiredis && $(MAKE) clean) > /dev/null || true + -(cd linenoise && $(MAKE) clean) > /dev/null || true + -(cd lua && $(MAKE) clean) > /dev/null || true + -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true + -(rm -f .make-arch) + +hiredis: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)hiredis$(ENDCOLOR) + cd hiredis && $(MAKE) static ARCH="$(ARCH)" + +linenoise: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) + cd linenoise && $(MAKE) ARCH="$(ARCH)" + +lua: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)lua$(ENDCOLOR) + cd lua && $(MAKE) CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(ARCH)" ansi + +jemalloc: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)jemalloc$(ENDCOLOR) + cd jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a + +.PHONY: default conditional_clean hiredis linenoise lua jemalloc diff --git a/src/Makefile b/src/Makefile index 76bc223b..f75e43ca 100644 --- a/src/Makefile +++ b/src/Makefile @@ -5,8 +5,7 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') OPTIMIZATION?=-O2 - -LUA_CFLAGS=-O2 -Wall +DEPENDENCY_TARGETS=hiredis linenoise lua ifeq ($(uname_S),Linux) ifneq ($(FORCE_LIBC_MALLOC),yes) @@ -15,8 +14,6 @@ ifeq ($(uname_S),Linux) endif ifeq ($(uname_S),SunOS) - # make isinf() available - LUA_CFLAGS+=-D__C99FEATURES__=1 CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 CCLINK?=-ldl -lnsl -lsocket -lm -lpthread DEBUG?=-g -ggdb @@ -27,27 +24,23 @@ else endif ifeq ($(USE_TCMALLOC),yes) - ALLOD_DEPS= ALLOC_LINK=-ltcmalloc ALLOC_FLAGS=-DUSE_TCMALLOC endif ifeq ($(USE_TCMALLOC_MINIMAL),yes) - ALLOD_DEPS= ALLOC_LINK=-ltcmalloc_minimal ALLOC_FLAGS=-DUSE_TCMALLOC endif ifeq ($(USE_JEMALLOC),yes) - ALLOC_DEP=../deps/jemalloc/lib/libjemalloc.a - ALLOC_LINK=$(ALLOC_DEP) -ldl + ALLOC_LINK=../deps/jemalloc/lib/libjemalloc.a -ldl ALLOC_FLAGS=-DUSE_JEMALLOC -I../deps/jemalloc/include + DEPENDENCY_TARGETS+= jemalloc endif CCLINK+= $(ALLOC_LINK) CFLAGS+= $(ALLOC_FLAGS) -LUA_CFLAGS+= $(ARCH) - CCOPT= $(CFLAGS) $(ARCH) $(PROF) PREFIX= /usr/local @@ -160,52 +153,52 @@ ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endian.h zipmap.o: zipmap.c zmalloc.h endian.h zmalloc.o: zmalloc.c config.h zmalloc.h -.PHONY: dependencies +# Clean local objects when ARCH is different +ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) +.make-arch: clean +else +.make-arch: +endif -dependencies: - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)hiredis$(ENDCOLOR) - @cd ../deps/hiredis && $(MAKE) static ARCH="$(ARCH)" - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) - @cd ../deps/linenoise && $(MAKE) ARCH="$(ARCH)" - @echo $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)Lua ansi$(ENDCOLOR) - @cd ../deps/lua && $(MAKE) CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(ARCH)" ansi +.make-arch: + -(cd ../deps && make $(DEPENDENCY_TARGETS) ARCH="$(ARCH)") + -(echo $(ARCH) > .make-arch) -../deps/jemalloc/lib/libjemalloc.a: - cd ../deps/jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a - -redis-server: dependencies $(OBJ) +redis-server: .make-arch $(OBJ) $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) ../deps/lua/src/liblua.a $(CCLINK) -redis-benchmark: dependencies $(BENCHOBJ) - @cd ../deps/hiredis && $(MAKE) static +redis-benchmark: .make-arch $(BENCHOBJ) $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) -redis-benchmark.o: +redis-benchmark.o: redis-benchmark.c .make-arch $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $< -redis-cli: dependencies $(CLIOBJ) +redis-cli: .make-arch $(CLIOBJ) $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) -redis-cli.o: +redis-cli.o: redis-cli.c .make-arch $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis -I../deps/linenoise $(DEBUG) $(COMPILE_TIME) $< -redis-check-dump: $(CHECKDUMPOBJ) +redis-check-dump: .make-arch $(CHECKDUMPOBJ) $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) -redis-check-aof: $(CHECKAOFOBJ) +redis-check-aof: .make-arch $(CHECKAOFOBJ) $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) # Because the jemalloc.h header is generated as a part of the jemalloc build -# process, building it should complete before building any other object. -%.o: %.c $(ALLOC_DEP) +# process, building it should complete before building any other object. Instead of +# depending on a single artifact, simply build all dependencies first. +%.o: %.c .make-arch $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< +.PHONY: all clean distclean + clean: rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov - cd ../deps/hiredis && $(MAKE) $@ - cd ../deps/linenoise && $(MAKE) $@ - cd ../deps/lua && $(MAKE) $@ - -(cd ../deps/jemalloc && $(MAKE) distclean) + +distclean: clean + -(cd ../deps && $(MAKE) distclean) + -(rm -f .make-arch) dep: $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise From 5bb2c88e9414df90f029b88a38a67f5fc1e36219 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 15 Nov 2011 13:09:31 -0800 Subject: [PATCH 4/5] Rebuild source when allocator changes To do so, the Makefile stores the contents of the MALLOC environment variable in a file named .make-malloc. When the contents of this file and the MALLOC variable are not equal, it forces a rebuild of the Redis source tree. A side-effect of this change is that choosing an allocator can now be done using the single MALLOC variable instead of USE_TCMALLOC, USE_JEMALLOC and so forth. These variables continue to work for backwards compatibility. --- src/Makefile | 61 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/src/Makefile b/src/Makefile index f75e43ca..bedc5026 100644 --- a/src/Makefile +++ b/src/Makefile @@ -7,12 +7,6 @@ uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') OPTIMIZATION?=-O2 DEPENDENCY_TARGETS=hiredis linenoise lua -ifeq ($(uname_S),Linux) - ifneq ($(FORCE_LIBC_MALLOC),yes) - USE_JEMALLOC=yes - endif -endif - ifeq ($(uname_S),SunOS) CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 CCLINK?=-ldl -lnsl -lsocket -lm -lpthread @@ -23,17 +17,37 @@ else DEBUG?=-g -rdynamic -ggdb endif +# Default allocator +ifeq ($(uname_S),Linux) + MALLOC?=jemalloc +else + MALLOC?=libc +endif + +# Backwards compatibility for selecting an allocator ifeq ($(USE_TCMALLOC),yes) + MALLOC=tcmalloc +endif + +ifeq ($(USE_TCMALLOC_MINIMAL),yes) + MALLOC=tcmalloc_minimal +endif + +ifeq ($(USE_JEMALLOC),yes) + MALLOC=jemalloc +endif + +ifeq ($(MALLOC),tcmalloc) ALLOC_LINK=-ltcmalloc ALLOC_FLAGS=-DUSE_TCMALLOC endif -ifeq ($(USE_TCMALLOC_MINIMAL),yes) +ifeq ($(MALLOC),tcmalloc_minimal) ALLOC_LINK=-ltcmalloc_minimal ALLOC_FLAGS=-DUSE_TCMALLOC endif -ifeq ($(USE_JEMALLOC),yes) +ifeq ($(MALLOC),jemalloc) ALLOC_LINK=../deps/jemalloc/lib/libjemalloc.a -ldl ALLOC_FLAGS=-DUSE_JEMALLOC -I../deps/jemalloc/include DEPENDENCY_TARGETS+= jemalloc @@ -164,31 +178,44 @@ endif -(cd ../deps && make $(DEPENDENCY_TARGETS) ARCH="$(ARCH)") -(echo $(ARCH) > .make-arch) -redis-server: .make-arch $(OBJ) +# Clean local objects when allocator changes +ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc'), $(MALLOC)) +.make-malloc: clean +else +.make-malloc: +endif + +.make-malloc: + -(echo $(MALLOC) > .make-malloc) + +# Union of prerequisites +.prerequisites: .make-arch .make-malloc + +redis-server: .prerequisites $(OBJ) $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) ../deps/lua/src/liblua.a $(CCLINK) -redis-benchmark: .make-arch $(BENCHOBJ) +redis-benchmark: .prerequisites $(BENCHOBJ) $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) -redis-benchmark.o: redis-benchmark.c .make-arch +redis-benchmark.o: redis-benchmark.c .prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $< -redis-cli: .make-arch $(CLIOBJ) +redis-cli: .prerequisites $(CLIOBJ) $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) -redis-cli.o: redis-cli.c .make-arch +redis-cli.o: redis-cli.c .prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis -I../deps/linenoise $(DEBUG) $(COMPILE_TIME) $< -redis-check-dump: .make-arch $(CHECKDUMPOBJ) +redis-check-dump: .prerequisites $(CHECKDUMPOBJ) $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) -redis-check-aof: .make-arch $(CHECKAOFOBJ) +redis-check-aof: .prerequisites $(CHECKAOFOBJ) $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) # Because the jemalloc.h header is generated as a part of the jemalloc build # process, building it should complete before building any other object. Instead of # depending on a single artifact, simply build all dependencies first. -%.o: %.c .make-arch +%.o: %.c .prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< .PHONY: all clean distclean @@ -198,7 +225,7 @@ clean: distclean: clean -(cd ../deps && $(MAKE) distclean) - -(rm -f .make-arch) + -(rm -f .make-arch .make-malloc) dep: $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise From a63a3d4ddb992747020bd48f00c35158d7a447d6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 15 Nov 2011 13:28:57 -0800 Subject: [PATCH 5/5] Update README on allocators --- README | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README b/README index e3054887..3920b534 100644 --- a/README +++ b/README @@ -26,18 +26,19 @@ NOTE: if after building Redis with a 32 bit target you need to rebuild it Allocator --------- -By default Redis compiles and links against jemalloc under Linux, since -glibc malloc() has memory fragmentation problems. +Selecting a non-default memory allocator when building Redis is done by setting +the `MALLOC` environment variable. Redis is compiled and linked against libc +malloc by default, with the exception of jemalloc being the default on Linux +systems. This default was picked because jemalloc has proven to have fewer +fragmentation problems than libc malloc. -To force a libc malloc() build use: +To force compiling against libc malloc, use: - % make FORCE_LIBC_MALLOC=yes + % make MALLOC=libc -In all the other non Linux systems the libc malloc() is used by default. +To compile against jemalloc on Mac OS X systems, use: -On Mac OS X you can force a jemalloc based build using the following: - - % make USE_JEMALLOC=yes + % make MALLOC=jemalloc Verbose build -------------