mongodb – put things down

Today I was working in a MongoDB 3.0 sharded cluster environment. There was a particular sharded collection that had 300-some-odd chunks evenly distributed within each shard (thanks to the balancer). These chunks happened to be empty, and in need of some pre-splitting for near-future use.

I ended up writing MongoDB shell scripts to handle the migration of all the chunks to the primary shard, and to merge all of the chunks to a single chunk. The scripts adhere to the following:

Authenticate a clusterAdmin user against the admin database. (I actually used a “root” role user.)
Read the config database for sharding topology and chunk distribution.
Any “write”-like commands use sharding helpers where possible, and runCommand otherwise. No “write”-like commands use CRUD operations on the config database.

Don’t forget to stop the balancer before running these scripts, and then start the balancer when they’re done.

Step 1: Migrate Chunks to Primary Shard

It is impossible to merge chunks that are not on the same shard. First, it is necessary to migrate all the chunks, and for the sake of some standard, on the primary shard.

var databaseName = 'database'; // your database name
var collectionName = 'collection'; // your collection
var adminUsername = 'username'; // user with clusterAdmin role
var adminPassword = 'password'; // that user's password

var namespace = databaseName+'.'+collectionName;
var admindb = db.getSiblingDB('admin');
var configdb = db.getSiblingDB('config');

admindb.auth(adminUsername,adminPassword);

if ( sh.getBalancerState() ) {
	print('balancer is enabled, turn it off');
	quit();
}

var foundDatabase = configdb.databases.findOne({_id:databaseName,partitioned:true});
if ( !foundDatabase || !foundDatabase.primary ) {
	print('no partitioned database found with name '+databaseName);
	quit();
}

var primaryShard = foundDatabase.primary;
print('primary shard is '+primaryShard);

var eligibleChunks = configdb.chunks.find({ns:namespace,shard:{$ne:primaryShard}});
if ( eligibleChunks.count() === 0 ) {
	print('no eligible chunks were found, either none exist or they are all already on the primary shard.');
}

var databaseName = 'database'; // your database name

var collectionName = 'collection'; // your collection

var adminUsername = 'username'; // user with clusterAdmin role

var adminPassword = 'password'; // that user's password

var namespace = databaseName+'.'+collectionName;

var admindb = db.getSiblingDB('admin');

var configdb = db.getSiblingDB('config');

admindb.auth(adminUsername,adminPassword);

if ( sh.getBalancerState() ) {

print('balancer is enabled, turn it off');

quit();

}

var foundDatabase = configdb.databases.findOne({_id:databaseName,partitioned:true});

if ( !foundDatabase || !foundDatabase.primary ) {

print('no partitioned database found with name '+databaseName);

quit();

}

var primaryShard = foundDatabase.primary;

print('primary shard is '+primaryShard);

var eligibleChunks = configdb.chunks.find({ns:namespace,shard:{$ne:primaryShard}});

if ( eligibleChunks.count() === 0 ) {

print('no eligible chunks were found, either none exist or they are all already on the primary shard.');

}

var i = 0;

eligibleChunks.forEach(function(chunk) {

i++;

print(i+' moving chunk id '+chunk._id.toString());

var result = admindb.runCommand({moveChunk:namespace,bounds:[chunk.min,chunk.max],to:primaryShard});

if ( !result || !result.ok ) {

print(i+' error moving chunk with id '+chunk._id.toString());

print(JSON.stringify(result));

quit();

}

print(i+' moved chunk successfully');

});

Run the script as follows:

1	mongo mongos.example.com/admin migratechunks.js

Remember, authentication happens in the script, so no need to pass -u or -p credentials via the CLI. The script automatically finds the admin and config databases as well, so connecting to the admin database is not required.

Step 2: Merge All Chunks

Now that all the chunks are on the primary shard, it is time to merge them into a single chunk. Only contiguous chunks can be merged, and only two chunks at a time can be considered. If there are hundreds or thousands of these 64MB chunks, this can be tedious if done manually. The following MongoDB shell script finds a pair of contiguous chunks and merges them; it repeats this process until there are no more chunks left to merge.

var namespace = databaseName+'.'+collectionName;
var admindb = db.getSiblingDB('admin');
var configdb = db.getSiblingDB('config');

admindb.auth(adminUsername,adminPassword);

if ( sh.getBalancerState() ) {
	print('balancer is enabled, turn it off');
	quit();
}

var databaseName = 'database'; // your database name

var collectionName = 'collection'; // your collection

var adminUsername = 'username'; // user with clusterAdmin role

var adminPassword = 'password'; // that user's password

var namespace = databaseName+'.'+collectionName;

var admindb = db.getSiblingDB('admin');

var configdb = db.getSiblingDB('config');

admindb.auth(adminUsername,adminPassword);

if ( sh.getBalancerState() ) {

print('balancer is enabled, turn it off');

quit();

}

var foundDatabase = configdb.databases.findOne({_id:databaseName,partitioned:true});

if ( !foundDatabase || !foundDatabase.primary ) {

print('no partitioned database found with name '+databaseName);

quit();

}

var eligibleChunks = 0;

var i = 0;

while ( eligibleChunks = configdb.chunks.count({ns:namespace}) ) {

i++;

print(i+' eligible chunks = '+eligibleChunks);

print(i+' finding subject chunk');

var subject = configdb.chunks.findOne({ns:namespace});

if ( !subject || !subject.max || !subject.min ) {

print(i+' no valid subject chunk found');

print(subject);

break;

}

print(i+' finding contiguous chunk');

var contiguous = configdb.chunks.findOne({ns:namespace,min:subject.max});

if ( !contiguous || !contiguous.max || !contiguous.min ) {

print(i+' no valid contiguous chunk found');

print(contiguous);

break;

}

print(i+' merging chunks');

var result = admindb.runCommand({mergeChunks:namespace,bounds:[

subject.min,

contiguous.max

]});

if ( !result || !result.ok ) {

print(i+' merge failure');

print(JSON.stringify(result));

break;

}

print(i+' merged '+JSON.stringify(subject.min)+', '+JSON.stringify(contiguous.max));

}

Just like with the previous migrate script, run this one the same way:

1	mongo mongos.example.com/admin mergechunks.js

Further Considerations

This exercise is also beneficial as prep work for:

Unsharding a collection
Unsharding a database
Changing the shard key of a sharded collection
Manually re/pre-splitting of shard chunks

Premise

When installing mongos (not mongod) on CentOS 6 using the RPMs provided in MongoDB’s repository, no startup/service/init.d scripts are created, or really anything for that matter, to aid in managing mongos as a service. All the RPM provides is the binary.

If you want to treat mongos as a service, there are quite a few steps to follow beyond just setting up a script in init.d:

mongos YAML configuration file
mongos sysconfig service overrides
SELinux port definition
a user to run mongos

Most of the time there are many mongos processes distributed within a MongoDB cluster. Configuring each of these can be a hassle, let alone installing and managing them, unless you use an automation tool.

Be sure to read “Install MongoDB on Red Hat Enterprise or CentOS Linux” first.

mongos Service Scripts

Below are the 3 files I use when preparing a mongos cluster:

The init.d service script for managing the mongos process
A base mongos configuration file in YAML
An automation script to prepare the CentOS 6 environment for mongos

/etc/init.d/mongos

The following init.d script has been retooled from the stock MongoDB 3.0 mongod for mongos. Watch out for the subtle differences.

#!/bin/bash

# mongos - Startup script for mongos

# chkconfig: 35 85 15
# description: Mongo is a scalable, document-oriented database.
# processname: mongos
# config: /etc/mongos.conf
# pidfile: /var/run/mongodb/mongos.pid

. /etc/rc.d/init.d/functions

# things from mongos.conf get there by mongos reading it

# NOTE: if you change any OPTIONS here, you get what you pay for:
# this script assumes all options are in the config file.
CONFIGFILE="/etc/mongos.conf"
OPTIONS=" -f $CONFIGFILE"
SYSCONFIG="/etc/sysconfig/mongos"
LOCKFILE="/var/lock/subsys/mongos"

PIDFILEPATH=`awk -F'[:=]' -v IGNORECASE=1 '/^[[:blank:]]*(processManagement\.)?pidfilepath[[:blank:]]*[:=][[:blank:]]*/{print $2}' "$CONFIGFILE" | tr -d "[:blank:]\"'"`

mongos=${MONGOS-/usr/bin/mongos}

MONGO_USER=mongod
MONGO_GROUP=mongod

if [ -f "$SYSCONFIG" ]; then
    . "$SYSCONFIG"
fi

PIDDIR=`dirname $PIDFILEPATH`

# Handle NUMA access to CPUs (SERVER-3574)
# This verifies the existence of numactl as well as testing that the command works
NUMACTL_ARGS="--interleave=all"
if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null 2>/dev/null
then
    NUMACTL="numactl $NUMACTL_ARGS"
else
    NUMACTL=""
fi

start()
{
  # Make sure the default pidfile directory exists
  if [ ! -d $PIDDIR ]; then
    install -d -m 0755 -o $MONGO_USER -g $MONGO_GROUP $PIDDIR
  fi

# Recommended ulimit values for mongod or mongos
  # See http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings
  #
  ulimit -f unlimited
  ulimit -t unlimited
  ulimit -v unlimited
  ulimit -n 64000
  ulimit -m unlimited
  ulimit -u 32000

echo -n $"Starting mongos: "
  daemon --user "$MONGO_USER" --check $mongos "$NUMACTL $mongos $OPTIONS >/dev/null 2>&1"
  RETVAL=$?
  echo
  [ $RETVAL -eq 0 ] && touch "$LOCKFILE"
}

stop()
{
  echo -n $"Stopping mongos: "
  mongo_killproc "$PIDFILEPATH" $mongos
  RETVAL=$?
  echo
  [ $RETVAL -eq 0 ] && rm -f "$LOCKFILE"
}

restart () {
	stop
	start
}

# Send TERM signal to process and wait up to 300 seconds for process to go away.
# If process is still alive after 300 seconds, send KILL signal.
# Built-in killproc() (found in /etc/init.d/functions) is on certain versions of Linux
# where it sleeps for the full $delay seconds if process does not respond fast enough to
# the initial TERM signal.
mongo_killproc()
{
  local pid_file=$1
  local procname=$2
  local -i delay=300
  local -i duration=10
  local pid=`pidofproc -p "${pid_file}" ${procname}`

kill -TERM $pid >/dev/null 2>&1
  usleep 100000
  local -i x=0
  while [ $x -le $delay ] && checkpid $pid; do
    sleep $duration
    x=$(( $x + $duration))
  done

kill -KILL $pid >/dev/null 2>&1
  usleep 100000

rm -f "${pid_file}"

checkpid $pid
  local RC=$?
  [ "$RC" -eq 0 ] && failure "${procname} shutdown" || success "${procname} shutdown"
  RC=$((! $RC))
  return $RC
}

RETVAL=0

case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart|reload|force-reload)
    restart
    ;;
  condrestart)
    [ -f "$LOCKFILE" ] && restart || :
    ;;
  status)
    status $mongos
    RETVAL=$?
    ;;
  *)
    echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
    RETVAL=1
esac

exit $RETVAL

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

#!/bin/bash

# mongos - Startup script for mongos

# chkconfig: 35 85 15

# description: Mongo is a scalable, document-oriented database.

# processname: mongos

# config: /etc/mongos.conf

# pidfile: /var/run/mongodb/mongos.pid

. /etc/rc.d/init.d/functions

# things from mongos.conf get there by mongos reading it

# NOTE: if you change any OPTIONS here, you get what you pay for:

# this script assumes all options are in the config file.

CONFIGFILE="/etc/mongos.conf"

OPTIONS=" -f $CONFIGFILE"

SYSCONFIG="/etc/sysconfig/mongos"

LOCKFILE="/var/lock/subsys/mongos"

PIDFILEPATH=`awk -F'[:=]' -v IGNORECASE=1 '/^[[:blank:]]*(processManagement\.)?pidfilepath[[:blank:]]*[:=][[:blank:]]*/{print $2}' "$CONFIGFILE" | tr -d "[:blank:]\"'"`

mongos=${MONGOS-/usr/bin/mongos}

MONGO_USER=mongod

MONGO_GROUP=mongod

if [ -f "$SYSCONFIG" ]; then

. "$SYSCONFIG"

PIDDIR=`dirname $PIDFILEPATH`

# Handle NUMA access to CPUs (SERVER-3574)

# This verifies the existence of numactl as well as testing that the command works

NUMACTL_ARGS="--interleave=all"

if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null 2>/dev/null

then

NUMACTL="numactl $NUMACTL_ARGS"

else

NUMACTL=""

start()

{

# Make sure the default pidfile directory exists

if [ ! -d $PIDDIR ]; then

install -d -m 0755 -o $MONGO_USER -g $MONGO_GROUP $PIDDIR

# Recommended ulimit values for mongod or mongos

# See http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings

ulimit -f unlimited

ulimit -t unlimited

ulimit -v unlimited

ulimit -n 64000

ulimit -m unlimited

ulimit -u 32000

echo -n $"Starting mongos: "

daemon --user "$MONGO_USER" --check $mongos "$NUMACTL $mongos $OPTIONS >/dev/null 2>&1"

RETVAL=$?

echo

[ $RETVAL -eq 0 ] && touch "$LOCKFILE"

}

stop()

{

echo -n $"Stopping mongos: "

mongo_killproc "$PIDFILEPATH" $mongos

RETVAL=$?

echo

[ $RETVAL -eq 0 ] && rm -f "$LOCKFILE"

}

restart () {

stop

start

}

# Send TERM signal to process and wait up to 300 seconds for process to go away.

# If process is still alive after 300 seconds, send KILL signal.

# Built-in killproc() (found in /etc/init.d/functions) is on certain versions of Linux

# where it sleeps for the full $delay seconds if process does not respond fast enough to

# the initial TERM signal.

mongo_killproc()

{

local pid_file=$1

local procname=$2

local -i delay=300

local -i duration=10

local pid=`pidofproc -p "${pid_file}" ${procname}`

kill -TERM $pid >/dev/null 2>&1

usleep 100000

local -i x=0

while [ $x -le $delay ] && checkpid $pid; do

sleep $duration

x=$(( $x + $duration))

done

kill -KILL $pid >/dev/null 2>&1

usleep 100000

rm -f "${pid_file}"

checkpid $pid

local RC=$?

[ "$RC" -eq 0 ] && failure "${procname} shutdown" || success "${procname} shutdown"

RC=$((! $RC))

return $RC

}

RETVAL=0

case "$1" in

start)

start

;;

stop)

stop

;;

restart|reload|force-reload)

restart

;;

condrestart)

[ -f "$LOCKFILE" ] && restart || :

;;

status)

status $mongos

RETVAL=$?

;;

echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"

RETVAL=1

esac

exit $RETVAL

/etc/mongos.conf Base YAML Configuration

Be sure to set sharding.configDB appropriately.

# mongos base config in YAML

systemLog:

destination: "file"

path: "/var/log/mongodb/mongos.log"

logAppend: true

processManagement:

fork: true

pidFilePath: "/var/run/mongodb/mongos.pid"

net:

bindIp: "0.0.0.0"

port: 27017

sharding:

configDB: "localhost:27019"

Post-Install Environment Setup

Provided mongodb-org-mongos has already been installed, the following script will do most of the legwork to get the CentOS 6 environment up and running. Note: The script is pulling the configuration file and service file dependencies from a web server using wget; this should be tailored. It performs the following:

Add a mongod user (yes, with a d) with the same settings that would be supplied from mongodb-org-server.
Add a placeholder file for sysconfig.
Add the mongos.conf file with the base configuration noted in previous (wget).
Add the init.d service script (wget).
If SELinux is enabled, set the contexts for the files created in previous, and add the port definition via semanage. Note that it will automatically install the required tools to perform this task if required.
Add and enable the new mongos service

#!/bin/bash

append='[mongos-prep] ';

SYSCONFIG_FILE="/etc/sysconfig/mongos";
ETCCONFIG_FILE="/etc/mongos.conf";
INITD_FILE="/etc/init.d/mongos";
LOGDIR_PATH="/var/log/mongodb";
INITD_DOWNLOAD="http://host.tld/path/to/init.d/mongos";
ETCCONFIG_DOWNLOAD="http://host.tld/path/to/mongos.conf";

echo "${append}Adding mongod user";
useradd -l -M -r -U -s /bin/false -d /var/lib/mongo mongod;

echo "${append}Adding sysconfig file";
echo '# TODO: add relevant configuration stuff here.' >> "${SYSCONFIG_FILE}";
chmod 0644 "${SYSCONFIG_FILE}";

echo "${append}Downloading and installing init.d script...";
wget "${INITD_DOWNLOAD}" -O "${INITD_FILE}";
chmod 0755 "${INITD_FILE}";

echo "${append}Adding regular config file ${ETCCONFIG_FILE}";
wget "${ETCCONFIG_DOWNLOAD}" -O "${ETCCONFIG_FILE}";
chmod 0644 "${ETCCONFIG_FILE}";

echo "${append}Preparing log directory";
mkdir -p "${LOGDIR_PATH}";

if selinuxenabled; then
	echo "${append}SELinux enabled, checking checking required tools...";
	if ! hash restorecon 2>/dev/null; then
		echo "${append}restorecon missing, installing policycoreutils...";
		sleep 2;
		yum -y install policycoreutils;
	fi;
	if ! hash semanage 2>/dev/null; then
		echo "${append}semanage missing, installing policycoreutils-python...";
		sleep 2;
		yum -y install policycoreutils-python;
	fi;
	
	echo "${append}Restoring SELinux contexts";
	restorecon -F "${INITD_FILE}";
	restorecon -F "${SYSCONFIG_FILE}";
	restorecon -F "${ETCCONFIG_FILE}";
	restorecon -F "${LOGDIR_PATH}";
	
	echo "${append}Adding port definition mongod_port_t on tcp/27017 to SELinux port, this will take a while...";
	semanage port -a -t mongod_port_t -p tcp 27017;

echo "${append}Adding file contexts...";
	semanage fcontext -a -t mongod_initrc_exec_t '/etc/rc\.d/init\.d/mongos';
	semanage fcontext -a -t mongod_exec_t '/usr/bin/mongos';
fi;

echo "${append}Adding and enabling mongos in the startup config";
chkconfig --add mongos;
chkconfig mongos on;

echo "${append}complete.";

#!/bin/bash

append='[mongos-prep] ';

SYSCONFIG_FILE="/etc/sysconfig/mongos";

ETCCONFIG_FILE="/etc/mongos.conf";

INITD_FILE="/etc/init.d/mongos";

LOGDIR_PATH="/var/log/mongodb";

INITD_DOWNLOAD="http://host.tld/path/to/init.d/mongos";

ETCCONFIG_DOWNLOAD="http://host.tld/path/to/mongos.conf";

echo "${append}Adding mongod user";

useradd -l -M -r -U -s /bin/false -d /var/lib/mongo mongod;

echo "${append}Adding sysconfig file";

echo '# TODO: add relevant configuration stuff here.' >> "${SYSCONFIG_FILE}";

chmod 0644 "${SYSCONFIG_FILE}";

echo "${append}Downloading and installing init.d script...";

wget "${INITD_DOWNLOAD}" -O "${INITD_FILE}";

chmod 0755 "${INITD_FILE}";

echo "${append}Adding regular config file ${ETCCONFIG_FILE}";

wget "${ETCCONFIG_DOWNLOAD}" -O "${ETCCONFIG_FILE}";

chmod 0644 "${ETCCONFIG_FILE}";

echo "${append}Preparing log directory";

mkdir -p "${LOGDIR_PATH}";

if selinuxenabled; then

echo "${append}SELinux enabled, checking checking required tools...";

if ! hash restorecon 2>/dev/null; then

echo "${append}restorecon missing, installing policycoreutils...";

sleep 2;

yum -y install policycoreutils;

fi;

if ! hash semanage 2>/dev/null; then

echo "${append}semanage missing, installing policycoreutils-python...";

sleep 2;

yum -y install policycoreutils-python;

fi;

echo "${append}Restoring SELinux contexts";

restorecon -F "${INITD_FILE}";

restorecon -F "${SYSCONFIG_FILE}";

restorecon -F "${ETCCONFIG_FILE}";

restorecon -F "${LOGDIR_PATH}";

echo "${append}Adding port definition mongod_port_t on tcp/27017 to SELinux port, this will take a while...";

semanage port -a -t mongod_port_t -p tcp 27017;

echo "${append}Adding file contexts...";

semanage fcontext -a -t mongod_initrc_exec_t '/etc/rc\.d/init\.d/mongos';

semanage fcontext -a -t mongod_exec_t '/usr/bin/mongos';

fi;

echo "${append}Adding and enabling mongos in the startup config";

chkconfig --add mongos;

chkconfig mongos on;

echo "${append}complete.";

SELinux and MongoDB

The automation script above does not add all of the required SELinux policies required for mongos. Install the mongodb-org package to initialize all of the policies; it can be removed later if need be while keeping these policies intact. Finding the script used by MongoDB (or making one up) is a task for another day. A policy dump from semanage is as follows:

semanage port -l | grep mongo

mongod_port_t tcp 27017, 27017-27019, 28017-28019

semanage fcontext -l | grep mongo

/etc/rc\.d/init\.d/mongod regular file system_u:object_r:mongod_initrc_exec_t:s0

/usr/bin/mongod regular file system_u:object_r:mongod_exec_t:s0

/usr/share/aeolus-conductor/dbomatic/dbomatic regular file system_u:object_r:mongod_exec_t:s0

/var/lib/mongodb(/.*)? all files system_u:object_r:mongod_var_lib_t:s0

/var/log/mongodb(/.*)? all files system_u:object_r:mongod_log_t:s0

/var/run/aeolus/dbomatic\.pid regular file system_u:object_r:mongod_var_run_t:s0

/var/run/mongodb(/.*)? all files system_u:object_r:mongod_var_run_t:s0

semanage permissive -l | grep mongo

mongo_t

Tag: mongodb

MongoDB: Migrate and Merge All Chunks in Shard

Step 1: Migrate Chunks to Primary Shard

Step 2: Merge All Chunks

Further Considerations

MongoDB 3.0: mongos CentOS 6 service scripts

Premise

mongos Service Scripts

/etc/init.d/mongos

/etc/mongos.conf Base YAML Configuration

Post-Install Environment Setup

SELinux and MongoDB