diff --git a/accumulo b/accumulo new file mode 100755 index 0000000..b056ee6 --- /dev/null +++ b/accumulo @@ -0,0 +1,127 @@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Guarantees that Accumulo and its environment variables are set. +# +# Values set by script that can be user provided. If not provided script attempts to infer. +# ACCUMULO_CONF_DIR Location where accumulo-env.sh, accumulo-site.xml and friends will be read from +# ACCUMULO_LOG_DIR Directory for Accumulo daemon logs +# JAVA_HOME Directory Java +# +# Values always set by script. +# MALLOC_ARENA_MAX To work around a memory management bug (see ACCUMULO-847) +# +# Values set by script if certain files exist +# ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins +# ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers + +# Set the configuration directory +export ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR:-/etc/accumulo} +if [[ -z $ACCUMULO_CONF_DIR || ! -d $ACCUMULO_CONF_DIR || ! -r $ACCUMULO_CONF_DIR || ! -x $ACCUMULO_CONF_DIR ]]; then + echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists and can be read." 1>&2 + exit 1 +fi +export ACCUMULO_HOME="${ACCUMULO_HOME:-$ACCUMULO_CONF_DIR}" + +# Read environment configuration +if [[ -f "$ACCUMULO_CONF_DIR/accumulo-env.sh" ]]; then + . "$ACCUMULO_CONF_DIR/accumulo-env.sh" +fi + +# Set the log directory +export ACCUMULO_LOG_DIR=${ACCUMULO_LOG_DIR:-/var/log/accumulo} +if [[ -z $ACCUMULO_LOG_DIR || ! -d $ACCUMULO_LOG_DIR || ! -w $ACCUMULO_LOG_DIR || ! -x $ACCUMULO_LOG_DIR ]]; then + echo "ACCUMULO_LOG_DIR=$ACCUMULO_LOG_DIR is not a valid directory. Please make sure it exists and can be written. Or, override this in $ACCUMULO_CONF_DIR/accumulo-env.sh" 1>&2 + exit 1 +fi + +# See HADOOP-7154 and ACCUMULO-847 +export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1} + +# ACCUMULO-1985 provide a way to use the scripts and still bind to all network interfaces +export ACCUMULO_MONITOR_BIND_ALL=${ACCUMULO_MONITOR_BIND_ALL:-"false"} + +# Check for jaas.conf configuration +if [[ -z $ACCUMULO_JAAS_CONF && -f "$ACCUMULO_CONF_DIR/jaas.conf" ]]; then + export ACCUMULO_JAAS_CONF=${ACCUMULO_CONF_DIR}/jaas.conf +fi +if [[ -n $ACCUMULO_JAAS_CONF ]]; then + export ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.auth.login.config=${ACCUMULO_JAAS_CONF}" +fi + +# Check for krb5.conf configuration +if [[ -z $ACCUMULO_KRB5_CONF && -f "${ACCUMULO_CONF_DIR}/krb5.conf" ]]; then + export ACCUMULO_KRB5_CONF=${ACCUMULO_CONF_DIR}/krb5.conf +fi +if [[ -n $ACCUMULO_KRB5_CONF ]]; then + export ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.krb5.conf=${ACCUMULO_KRB5_CONF}" +fi + +# +# Add appropriate options for process type +# +case "$1" in +master) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_MASTER_OPTS}" ;; +gc) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_GC_OPTS}" ;; +tserver) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_TSERVER_OPTS}" ;; +monitor) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_MONITOR_OPTS}" ;; +logger) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_LOGGER_OPTS}" ;; +*) export ACCUMULO_OPTS="${ACCUMULO_GENERAL_OPTS} ${ACCUMULO_OTHER_OPTS}" ;; +esac + +function getClasspath() { + /usr/bin/build-classpath \ + accumulo \ + accumulo/accumulo-tserver \ + apache-commons-cli \ + apache-commons-codec \ + apache-commons-collections \ + apache-commons-configuration \ + apache-commons-lang \ + apache-commons-logging \ + apache-commons-math \ + apache-commons-vfs \ + beust-jcommander \ + guava \ + hadoop/hadoop-auth \ + hadoop/hadoop-common \ + hadoop/hadoop-hdfs \ + jansi/jansi \ + jline/jline \ + libthrift \ + log4j-1.2.17 \ + slf4j/slf4j-api \ + slf4j/slf4j-log4j12 \ + zookeeper/zookeeper \ + ; +} +export CLASSPATH="${ACCUMULO_CONF_DIR}:$(getClasspath)" + +export JAVA_HOME=${JAVA_HOME:-$(dirname $(dirname $(readlink -ef $(which java))))} +export JAVA="${JAVA_HOME}/bin/java" + +# app isn't used anywhere, but it makes the process easier to spot when ps/top/snmp truncate the command line +# swap stderr and stdout to pipe stderr to logger, then move logger's echo to +# stdout, before swapping back, so the caller sees what they'd expect +($JAVA "-Dapp=$1" \ + $ACCUMULO_OPTS \ + -classpath "${CLASSPATH}" \ + -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" \ + -XX:-OmitStackTraceInFastThrow \ + -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl \ + org.apache.accumulo.start.Main \ + "$@" 3>&1 1>&2 2>&3 | logger -i -s 2>&1) 3>&1 1>&2 2>&3 diff --git a/accumulo.spec b/accumulo.spec index 39d0230..46e47a4 100644 --- a/accumulo.spec +++ b/accumulo.spec @@ -9,20 +9,22 @@ Name: %{proj} Version: 1.6.0 -Release: 2%{?dist} +Release: 3%{?dist} Summary: A software platform for processing vast amounts of data License: ASL 2.0 Group: Development/Libraries URL: http://%{name}.apache.org Source0: https://github.com/apache/%{name}/archive/%{commit}/%{name}-%{version}-%{shortcommit}.tar.gz +# startup script +Source1: %{name} # systemd service files -Source1: %{name}-master.service -Source2: %{name}-tserver.service -Source3: %{name}-gc.service -Source4: %{name}-tracer.service +Source2: %{name}-master.service +Source3: %{name}-tserver.service +Source4: %{name}-gc.service +Source5: %{name}-tracer.service %if %{include_monitor} -Source5: %{name}-monitor.service +Source6: %{name}-monitor.service %endif # Upstream patches needed for Fedora @@ -38,6 +40,7 @@ Patch6: commons-configuration.patch Patch7: commons-math.patch Patch8: native-code.patch Patch9: disabled-tests.patch +Patch10: default-conf.patch # Accumulo depends on Hadoop, and Hadoop is not built for ARM ExcludeArch: %{arm} @@ -293,6 +296,7 @@ This package contains the API documentation for %{longproj}. %patch7 -p1 %patch8 -p1 %patch9 -p1 +%patch10 -p1 # Update dependency versions %pom_xpath_set "pom:project/pom:dependencyManagement/pom:dependencies/pom:dependency[pom:artifactId='jline']/pom:version" "2.10" @@ -362,20 +366,13 @@ install -d -m 755 %{buildroot}%{_var}/cache/%{name} cp -af server/native/target/%{name}-native-%{version}/%{name}-native-%{version}/lib%{name}.so %{buildroot}%{_libdir}/%{name} # upstream scripts and config -install -d -m 755 %{buildroot}%{_libexecdir}/%{name} -cp -af bin/%{name} bin/config.sh %{buildroot}%{_libexecdir}/%{name} install -d -m 755 %{buildroot}%{_sysconfdir}/%{name} bin/bootstrap_config.sh -o -d %{buildroot}%{_sysconfdir}/%{name} -s 3GB -n -v 2 for x in gc masters monitor slaves tracers; do rm -f %{buildroot}%{_sysconfdir}/%{name}/$x; done # service shortcut files install -d -m 755 %{buildroot}%{_bindir} -cat <"%{name}" -#! /usr/bin/bash -[[ \$ACCUMULO_CONF_DIR ]] || export ACCUMULO_CONF_DIR=/%{_sysconfdir}/%{name} -%{_libexecdir}/%{name}/%{name} \$@ -EOF -install -p -m 755 %{name} %{buildroot}%{_bindir} +install -p -m 755 %{SOURCE1} %{buildroot}%{_bindir} %if %{include_monitor} for service in master tserver shell init admin gc monitor tracer classpath version rfile-info login-info zookeeper create-token info jar; do @@ -391,12 +388,12 @@ done # systemd services install -d -m 755 %{buildroot}%{_unitdir} -install -p -m 644 %{SOURCE1} %{buildroot}%{_unitdir}/%{name}-master.service -install -p -m 644 %{SOURCE2} %{buildroot}%{_unitdir}/%{name}-tserver.service -install -p -m 644 %{SOURCE3} %{buildroot}%{_unitdir}/%{name}-gc.service -install -p -m 644 %{SOURCE4} %{buildroot}%{_unitdir}/%{name}-tracer.service +install -p -m 644 %{SOURCE2} %{buildroot}%{_unitdir}/%{name}-master.service +install -p -m 644 %{SOURCE3} %{buildroot}%{_unitdir}/%{name}-tserver.service +install -p -m 644 %{SOURCE4} %{buildroot}%{_unitdir}/%{name}-gc.service +install -p -m 644 %{SOURCE5} %{buildroot}%{_unitdir}/%{name}-tracer.service %if %{include_monitor} -install -p -m 644 %{SOURCE5} %{buildroot}%{_unitdir}/%{name}-monitor.service +install -p -m 644 %{SOURCE6} %{buildroot}%{_unitdir}/%{name}-monitor.service %endif %files @@ -407,12 +404,9 @@ install -p -m 644 %{SOURCE5} %{buildroot}%{_unitdir}/%{name}-monitor.service %doc README %doc NOTICE %dir %{_javadir}/%{name} -%dir %{_libexecdir}/%{name} %if 0%{?fedora} > 20 %dir %{_mavenpomdir}/%{name} %endif -%{_libexecdir}/%{name}/%{name} -%{_libexecdir}/%{name}/config.sh %{_bindir}/%{name} %{_bindir}/%{name}-shell %{_bindir}/%{name}-classpath @@ -443,7 +437,9 @@ install -p -m 644 %{SOURCE5} %{buildroot}%{_unitdir}/%{name}-monitor.service %{_unitdir}/%{name}-master.service %files tserver -f .mfiles-tserver +%if 0%{?fedora} > 20 %dir %{_jnidir}/%{name} +%endif %{_bindir}/%{name}-tserver %{_unitdir}/%{name}-tserver.service @@ -504,7 +500,7 @@ install -p -m 644 %{SOURCE5} %{buildroot}%{_unitdir}/%{name}-monitor.service %endif %pre core -getent group %{name} >/dev/null || groupadd -r %{name} +getent group %{name} >/dev/null || /usr/sbin/groupadd -r %{name} getent passwd %{name} >/dev/null || /usr/sbin/useradd --comment "%{longproj}" --shell /sbin/nologin -M -r -g %{name} --home %{_var}/cache/%{name} %{name} %post master @@ -525,6 +521,10 @@ getent passwd %{name} >/dev/null || /usr/sbin/useradd --comment "%{longproj}" -- %endif %changelog +* Wed Jul 16 2014 Christopher Tubbs - 1.6.0-3 +- Fix broken service launch scripts +- Add conditional for lib directory to build for f20 + * Wed Jul 9 2014 Christopher Tubbs - 1.6.0-2 - Add conditional for pom directory to build for f20 - Remove fno-strict-aliasing flag based on upstream ACCUMULO-2762 diff --git a/default-conf.patch b/default-conf.patch new file mode 100644 index 0000000..2aca605 --- /dev/null +++ b/default-conf.patch @@ -0,0 +1,145 @@ +diff --git a/conf/templates/accumulo-env.sh b/conf/templates/accumulo-env.sh +index e136a3f..c1b0b72 100644 +--- a/conf/templates/accumulo-env.sh ++++ b/conf/templates/accumulo-env.sh +@@ -15,34 +15,8 @@ + # See the License for the specific language governing permissions and + # limitations under the License. + +-### +-### Configure these environment variables to point to your local installations. +-### +-### The functional tests require conditional values, so keep this style: +-### +-### test -z "$JAVA_HOME" && export JAVA_HOME=/usr/lib/jvm/java +-### +-### +-### Note that the -Xmx -Xms settings below require substantial free memory: +-### you may want to use smaller values, especially when running everything +-### on a single machine. +-### +-if [ -z "$HADOOP_HOME" ] +-then +- test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop +-else +- HADOOP_PREFIX="$HADOOP_HOME" +- unset HADOOP_HOME +-fi +- +-# hadoop-1.2: +-# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf" +-# hadoop-2.0: +-test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop" +- +-test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java +-test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper +-test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs ++test -z "$ACCUMULO_CONF_DIR" && export ACCUMULO_CONF_DIR=/etc/accumulo ++test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=/var/log/accumulo + if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ] + then + POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy" +@@ -56,11 +30,5 @@ test -z "$ACCUMULO_OTHER_OPTS" && export ACCUMULO_OTHER_OPTS="${otherHigh_othe + # what do when the JVM runs out of heap memory + export ACCUMULO_KILL_CMD='kill -9 %p' + +-### Optionally look for hadoop and accumulo native libraries for your +-### platform in additional directories. (Use DYLD_LIBRARY_PATH on Mac OS X.) +-### May not be necessary for Hadoop 2.x or using an RPM that installs to +-### the correct system library directory. +-# export LD_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${PLATFORM}:${LD_LIBRARY_PATH} +- + # Should the monitor bind to all network interfaces -- default: false + # export ACCUMULO_MONITOR_BIND_ALL="true" +diff --git a/conf/templates/accumulo-metrics.xml b/conf/templates/accumulo-metrics.xml +index 60f9f8d..f30b098 100644 +--- a/conf/templates/accumulo-metrics.xml ++++ b/conf/templates/accumulo-metrics.xml +@@ -23,7 +23,7 @@ + Metrics log directory + --> + +- ${ACCUMULO_HOME}/metrics ++ /var/log/accumulo/metrics + + + + ++ instance.volumes ++ file:///tmp/accumulo ++ comma separated list of dfs URIs ++ ++ ++ + instance.zookeeper.host + localhost:2181 + comma separated list of zookeeper servers +@@ -28,15 +28,6 @@ + + + +- logger.dir.walog +- walogs +- The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local +- filesystem. In 1.5 write-ahead logs are stored in DFS. When 1.5 is started for the first time it will copy any 1.4 +- write ahead logs into DFS. It is possible to specify a comma-separated list of directories. +- +- +- +- + instance.secret + DEFAULT + A secret unique to a given instance that all servers must know in order to communicate with one another. +@@ -89,44 +80,7 @@ + ${mvnProjBaseDir} + + general.classpaths +- +- +- +- $ACCUMULO_HOME/lib/accumulo-server.jar, +- $ACCUMULO_HOME/lib/accumulo-core.jar, +- $ACCUMULO_HOME/lib/accumulo-start.jar, +- $ACCUMULO_HOME/lib/accumulo-fate.jar, +- $ACCUMULO_HOME/lib/accumulo-proxy.jar, +- $ACCUMULO_HOME/lib/[^.].*.jar, +- +- +- +- $ZOOKEEPER_HOME/zookeeper[^.].*.jar, +- +- +- +- $HADOOP_CONF_DIR, +- +- +- +- $HADOOP_PREFIX/share/hadoop/common/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar, +- $HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar, +- /usr/lib/hadoop/[^.].*.jar, +- /usr/lib/hadoop/lib/[^.].*.jar, +- /usr/lib/hadoop-hdfs/[^.].*.jar, +- /usr/lib/hadoop-mapreduce/[^.].*.jar, +- /usr/lib/hadoop-yarn/[^.].*.jar, +- +- +- +- $HADOOP_PREFIX/[^.].*.jar, +- $HADOOP_PREFIX/lib/(?!slf4j)[^.].*.jar, +- +- +- ++ + Classpaths that accumulo checks for updates and class files. + +