ApplicationPoolServer.h

00001 /*
00002  *  Phusion Passenger - http://www.modrails.com/
00003  *  Copyright (C) 2008  Phusion
00004  *
00005  *  This program is free software; you can redistribute it and/or modify
00006  *  it under the terms of the GNU General Public License as published by
00007  *  the Free Software Foundation; version 2 of the License.
00008  *
00009  *  This program is distributed in the hope that it will be useful,
00010  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
00011  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00012  *  GNU General Public License for more details.
00013  *
00014  *  You should have received a copy of the GNU General Public License along
00015  *  with this program; if not, write to the Free Software Foundation, Inc.,
00016  *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
00017  */
00018 #ifndef _PASSENGER_APPLICATION_POOL_SERVER_H_
00019 #define _PASSENGER_APPLICATION_POOL_SERVER_H_
00020 
00021 #include <boost/shared_ptr.hpp>
00022 
00023 #include <sys/types.h>
00024 #include <sys/wait.h>
00025 #include <sys/socket.h>
00026 #include <cstdlib>
00027 #include <errno.h>
00028 #include <unistd.h>
00029 
00030 #include "MessageChannel.h"
00031 #include "ApplicationPool.h"
00032 #include "Application.h"
00033 #include "Exceptions.h"
00034 #include "Logging.h"
00035 
00036 namespace Passenger {
00037 
00038 using namespace std;
00039 using namespace boost;
00040 
00041 
00042 /**
00043  * Multi-process usage support for ApplicationPool.
00044  *
00045  * ApplicationPoolServer implements a client/server architecture for ApplicationPool.
00046  * This allows one to use ApplicationPool in a multi-process environment (unlike
00047  * StandardApplicationPool). The cache/pool data is stored in the server. Different
00048  * processes can then access the pool through the server.
00049  *
00050  * ApplicationPoolServer itself does not inherit ApplicationPool. Instead, it returns
00051  * an ApplicationPool object via the connect() call. For example:
00052  * @code
00053  *   // Create an ApplicationPoolServer.
00054  *   ApplicationPoolServer server(...);
00055  *   
00056  *   // Now fork a child process, like Apache's prefork MPM eventually will.
00057  *   pid_t pid = fork();
00058  *   if (pid == 0) {
00059  *       // Child process
00060  *       
00061  *       // Connect to the server. After connection, we have an ApplicationPool
00062  *       // object!
00063  *       ApplicationPoolPtr pool(server.connect());
00064  *
00065  *       // We don't need to connect to the server anymore, so we detach from it.
00066  *       // This frees up some resources, such as file descriptors.
00067  *       server.detach();
00068  *
00069  *       ApplicationPool::SessionPtr session(pool->get("/home/webapps/foo"));
00070  *       do_something_with(session);
00071  *
00072  *       _exit(0);
00073  *   } else {
00074  *       // Parent process
00075  *       waitpid(pid, NULL, 0);
00076  *   }
00077  * @endcode
00078  *
00079  * <h2>Implementation notes</h2>
00080  *
00081  * <h3>Separate server executable</h3>
00082  * The actual server is implemented in ApplicationPoolServerExecutable.cpp, this class is
00083  * just a convenience class for starting/stopping the server executable and connecting
00084  * to it.
00085  *
00086  * In the past, the server logic itself was implemented in this class. This implies that
00087  * the ApplicationPool server ran inside the Apache process. This presented us with several
00088  * problems:
00089  * - Because of the usage of threads in the ApplicationPool server, the Apache VM size would
00090  *   go way up. This gave people the (wrong) impression that Passenger uses a lot of memory,
00091  *   or that it leaks memory.
00092  * - Although it's not entirely confirmed, we suspect that it caused heap fragmentation as
00093  *   well. Apache allocates lots and lots of small objects on the heap, and ApplicationPool
00094  *   server isn't exactly helping. This too gave people the (wrong) impression that
00095  *   Passenger leaks memory.
00096  * - It would unnecessarily bloat the VM size of Apache worker processes.
00097  * - We had to resort to all kinds of tricks to make sure that fork()ing a process doesn't
00098  *   result in file descriptor leaks.
00099  * - Despite everything, there was still a small chance that file descriptor leaks would
00100  *   occur, and this could not be fixed. The reason for this is that the Apache control
00101  *   process may call fork() right after the ApplicationPool server has established a new
00102  *   connection with a client.
00103  *
00104  * Because of these problems, it was decided to split the ApplicationPool server to a
00105  * separate executable. This comes with no performance hit.
00106  *
00107  * <h3>Anonymous server socket</h3>
00108  * Notice that ApplicationPoolServer does do not use TCP sockets at all, or even named Unix
00109  * sockets, despite being a server that can handle multiple clients! So ApplicationPoolServer
00110  * will expose no open ports or temporary Unix socket files. Only child processes are able
00111  * to use the ApplicationPoolServer.
00112  *
00113  * This is implemented through anonymous Unix sockets (<tt>socketpair()</tt>) and file descriptor
00114  * passing. It allows one to emulate <tt>accept()</tt>. ApplicationPoolServer is connected to
00115  * the server executable through a Unix socket pair. connect() sends a connect request to the
00116  * server through that socket. The server will then create a new socket pair, and pass one of
00117  * them back. This new socket pair represents the newly established connection.
00118  *
00119  * @ingroup Support
00120  */
00121 class ApplicationPoolServer {
00122 private:
00123         /**
00124          * Contains data shared between RemoteSession and Client.
00125          * Since RemoteSession and Client have different life times, i.e. one may be
00126          * destroyed before the other, they both use a smart pointer that points to
00127          * a SharedData. This way, the SharedData object is only destroyed when
00128          * both the RemoteSession and the Client object has been destroyed.
00129          */
00130         struct SharedData {
00131                 /**
00132                  * The socket connection to the ApplicationPool server, as was
00133                  * established by ApplicationPoolServer::connect().
00134                  */
00135                 int server;
00136                 
00137                 ~SharedData() {
00138                         close(server);
00139                 }
00140         };
00141         
00142         typedef shared_ptr<SharedData> SharedDataPtr;
00143         
00144         /**
00145          * An Application::Session which works together with ApplicationPoolServer.
00146          */
00147         class RemoteSession: public Application::Session {
00148         private:
00149                 SharedDataPtr data;
00150                 int id;
00151                 int reader;
00152                 int writer;
00153                 pid_t pid;
00154         public:
00155                 RemoteSession(SharedDataPtr data, pid_t pid, int id, int reader, int writer) {
00156                         this->data = data;
00157                         this->pid = pid;
00158                         this->id = id;
00159                         this->reader = reader;
00160                         this->writer = writer;
00161                 }
00162                 
00163                 virtual ~RemoteSession() {
00164                         closeReader();
00165                         closeWriter();
00166                         MessageChannel(data->server).write("close", toString(id).c_str(), NULL);
00167                 }
00168                 
00169                 virtual int getReader() const {
00170                         return reader;
00171                 }
00172                 
00173                 virtual void closeReader() {
00174                         if (reader != -1) {
00175                                 close(reader);
00176                                 reader = -1;
00177                         }
00178                 }
00179                 
00180                 virtual int getWriter() const {
00181                         return writer;
00182                 }
00183                 
00184                 virtual void closeWriter() {
00185                         if (writer != -1) {
00186                                 close(writer);
00187                                 writer = -1;
00188                         }
00189                 }
00190                 
00191                 virtual pid_t getPid() const {
00192                         return pid;
00193                 }
00194         };
00195         
00196         /**
00197          * An ApplicationPool implementation that works together with ApplicationPoolServer.
00198          * It doesn't do much by itself, its job is mostly to forward queries/commands to
00199          * the server and returning the result. Most of the logic is in the server executable.
00200          */
00201         class Client: public ApplicationPool {
00202         private:
00203                 SharedDataPtr data;
00204                 
00205         public:
00206                 /**
00207                  * Create a new Client.
00208                  *
00209                  * @param sock The newly established socket connection with the ApplicationPoolServer.
00210                  */
00211                 Client(int sock) {
00212                         data = ptr(new SharedData());
00213                         data->server = sock;
00214                 }
00215                 
00216                 virtual void clear() {
00217                         MessageChannel channel(data->server);
00218                         channel.write("clear", NULL);
00219                 }
00220                 
00221                 virtual void setMaxIdleTime(unsigned int seconds) {
00222                         MessageChannel channel(data->server);
00223                         channel.write("setMaxIdleTime", toString(seconds).c_str(), NULL);
00224                 }
00225                 
00226                 virtual void setMax(unsigned int max) {
00227                         MessageChannel channel(data->server);
00228                         channel.write("setMax", toString(max).c_str(), NULL);
00229                 }
00230                 
00231                 virtual unsigned int getActive() const {
00232                         MessageChannel channel(data->server);
00233                         vector<string> args;
00234                         
00235                         channel.write("getActive", NULL);
00236                         channel.read(args);
00237                         return atoi(args[0].c_str());
00238                 }
00239                 
00240                 virtual unsigned int getCount() const {
00241                         MessageChannel channel(data->server);
00242                         vector<string> args;
00243                         
00244                         channel.write("getCount", NULL);
00245                         channel.read(args);
00246                         return atoi(args[0].c_str());
00247                 }
00248                 
00249                 virtual pid_t getSpawnServerPid() const {
00250                         MessageChannel channel(data->server);
00251                         vector<string> args;
00252                         
00253                         channel.write("getSpawnServerPid", NULL);
00254                         channel.read(args);
00255                         return atoi(args[0].c_str());
00256                 }
00257                 
00258                 virtual Application::SessionPtr get(const string &appRoot, bool lowerPrivilege = true,
00259                            const string &lowestUser = "nobody") {
00260                         MessageChannel channel(data->server);
00261                         vector<string> args;
00262                         int reader, writer;
00263                         
00264                         channel.write("get", appRoot.c_str(),
00265                                 (lowerPrivilege) ? "true" : "false",
00266                                 lowestUser.c_str(), NULL);
00267                         if (!channel.read(args)) {
00268                                 throw IOException("The ApplicationPool server unexpectedly closed the connection.");
00269                         }
00270                         if (args[0] == "ok") {
00271                                 reader = channel.readFileDescriptor();
00272                                 writer = channel.readFileDescriptor();
00273                                 return ptr(new RemoteSession(data, atoi(args[1]), atoi(args[2]), reader, writer));
00274                         } else if (args[0] == "SpawnException") {
00275                                 if (args[2] == "true") {
00276                                         string errorPage;
00277                                         
00278                                         if (!channel.readScalar(errorPage)) {
00279                                                 throw IOException("The ApplicationPool server unexpectedly closed the connection.");
00280                                         }
00281                                         throw SpawnException(args[1], errorPage);
00282                                 } else {
00283                                         throw SpawnException(args[1]);
00284                                 }
00285                         } else if (args[0] == "IOException") {
00286                                 throw IOException(args[1]);
00287                         } else {
00288                                 throw IOException("The ApplicationPool server returned an unknown message.");
00289                         }
00290                 }
00291         };
00292         
00293         
00294         static const int SERVER_SOCKET_FD = 3;
00295         
00296         string m_serverExecutable;
00297         string m_spawnServerCommand;
00298         string m_logFile;
00299         string m_environment;
00300         string m_rubyCommand;
00301         string m_user;
00302         
00303         /**
00304          * The PID of the ApplicationPool server process. If no server process
00305          * is running, then <tt>serverPid == 0</tt>.
00306          *
00307          * @invariant
00308          *    if serverPid == 0:
00309          *       serverSocket == -1
00310          */
00311         pid_t serverPid;
00312         
00313         /**
00314          * The connection to the ApplicationPool server process. If no server
00315          * process is running, then <tt>serverSocket == -1</tt>.
00316          *
00317          * @invariant
00318          *    if serverPid == 0:
00319          *       serverSocket == -1
00320          */
00321         int serverSocket;
00322         
00323         /**
00324          * Shutdown the currently running ApplicationPool server process.
00325          *
00326          * @pre serverSocket != -1 && serverPid != 0
00327          * @post serverSocket == -1 && serverPid == 0
00328          */
00329         void shutdownServer() {
00330                 time_t begin;
00331                 bool done;
00332                 int ret;
00333                 
00334                 do {
00335                         ret = close(serverSocket);
00336                 } while (ret == -1 && errno == EINTR);
00337                 
00338                 P_DEBUG("Waiting for existing ApplicationPoolServerExecutable to exit...");
00339                 begin = time(NULL);
00340                 while (!done && time(NULL) < begin + 5) {
00341                         done = waitpid(serverPid, NULL, WNOHANG) > 0;
00342                         usleep(100000);
00343                 }
00344                 if (done) {
00345                         P_DEBUG("ApplicationPoolServerExecutable exited.");
00346                 } else {
00347                         P_DEBUG("ApplicationPoolServerExecutable not exited in time. Killing it...");
00348                         kill(serverPid, SIGTERM);
00349                         waitpid(serverPid, NULL, 0);
00350                 }
00351                 serverSocket = -1;
00352                 serverPid = 0;
00353         }
00354         
00355         /**
00356          * Start an ApplicationPool server process. If there's already one running,
00357          * then the currently running one will be shutdown.
00358          *
00359          * @post serverSocket != -1 && serverPid != 0
00360          * @throw SystemException Something went wrong.
00361          */
00362         void restartServer() {
00363                 int fds[2];
00364                 pid_t pid;
00365                 
00366                 if (serverPid != 0) {
00367                         shutdownServer();
00368                 }
00369                 
00370                 if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) {
00371                         throw SystemException("Cannot create a Unix socket pair", errno);
00372                 }
00373                 
00374                 pid = fork();
00375                 if (pid == 0) { // Child process.
00376                         dup2(fds[0], 3);
00377                         
00378                         // Close all unnecessary file descriptors
00379                         for (long i = sysconf(_SC_OPEN_MAX) - 1; i > SERVER_SOCKET_FD; i--) {
00380                                 close(i);
00381                         }
00382                         
00383                         execlp(
00384                                 #if 0
00385                                         "valgrind",
00386                                         "valgrind",
00387                                 #else
00388                                         m_serverExecutable.c_str(),
00389                                 #endif
00390                                 m_serverExecutable.c_str(),
00391                                 m_spawnServerCommand.c_str(),
00392                                 m_logFile.c_str(),
00393                                 m_environment.c_str(),
00394                                 m_rubyCommand.c_str(),
00395                                 m_user.c_str(),
00396                                 NULL);
00397                         int e = errno;
00398                         fprintf(stderr, "*** Passenger ERROR: Cannot execute %s: %s (%d)\n",
00399                                 m_serverExecutable.c_str(), strerror(e), e);
00400                         fflush(stderr);
00401                         _exit(1);
00402                 } else if (pid == -1) { // Error.
00403                         close(fds[0]);
00404                         close(fds[1]);
00405                         throw SystemException("Cannot create a new process", errno);
00406                 } else { // Parent process.
00407                         close(fds[0]);
00408                         serverSocket = fds[1];
00409                         serverPid = pid;
00410                 }
00411         }
00412 
00413 public:
00414         /**
00415          * Create a new ApplicationPoolServer object.
00416          *
00417          * @param serverExecutable The filename of the ApplicationPool server
00418          *            executable to use.
00419          * @param spawnServerCommand The filename of the spawn server to use.
00420          * @param logFile Specify a log file that the spawn server should use.
00421          *            Messages on its standard output and standard error channels
00422          *            will be written to this log file. If an empty string is
00423          *            specified, no log file will be used, and the spawn server
00424          *            will use the same standard output/error channels as the
00425          *            current process.
00426          * @param environment The RAILS_ENV environment that all RoR applications
00427          *            should use. If an empty string is specified, the current value
00428          *            of the RAILS_ENV environment variable will be used.
00429          * @param rubyCommand The Ruby interpreter's command.
00430          * @param user The user that the spawn manager should run as. This
00431          *             parameter only has effect if the current process is
00432          *             running as root. If the empty string is given, or if
00433          *             the <tt>user</tt> is not a valid username, then
00434          *             the spawn manager will be run as the current user.
00435          * @throws SystemException An error occured while trying to setup the spawn server
00436          *            or the server socket.
00437          * @throws IOException The specified log file could not be opened.
00438          */
00439         ApplicationPoolServer(const string &serverExecutable,
00440                      const string &spawnServerCommand,
00441                      const string &logFile = "",
00442                      const string &environment = "production",
00443                      const string &rubyCommand = "ruby",
00444                      const string &user = "")
00445         : m_serverExecutable(serverExecutable),
00446           m_spawnServerCommand(spawnServerCommand),
00447           m_logFile(logFile),
00448           m_environment(environment),
00449           m_rubyCommand(rubyCommand),
00450           m_user(user) {
00451                 serverSocket = -1;
00452                 serverPid = 0;
00453                 restartServer();
00454         }
00455         
00456         ~ApplicationPoolServer() {
00457                 if (serverSocket != -1) {
00458                         shutdownServer();
00459                 }
00460         }
00461         
00462         /**
00463          * Connects to the server and returns a usable ApplicationPool object.
00464          * All cache/pool data of this ApplicationPool is actually stored on
00465          * the server and shared with other clients, but that is totally
00466          * transparent to the user of the ApplicationPool object.
00467          *
00468          * @warning
00469          * One may only use the returned ApplicationPool object for handling
00470          * one session at a time. For example, don't do stuff like this:
00471          * @code
00472          *   ApplicationPoolPtr pool = server.connect();
00473          *   Application::SessionPtr session1 = pool->get(...);
00474          *   Application::SessionPtr session2 = pool->get(...);
00475          * @endcode
00476          * Otherwise, a deadlock can occur under certain circumstances.
00477          * @warning
00478          * Instead, one should call connect() multiple times:
00479          * @code
00480          *   ApplicationPoolPtr pool1 = server.connect();
00481          *   Application::SessionPtr session1 = pool1->get(...);
00482          *   
00483          *   ApplicationPoolPtr pool2 = server.connect();
00484          *   Application::SessionPtr session2 = pool2->get(...);
00485          * @endcode
00486          *
00487          * @throws SystemException Something went wrong.
00488          * @throws IOException Something went wrong.
00489          */
00490         ApplicationPoolPtr connect() {
00491                 MessageChannel channel(serverSocket);
00492                 int clientConnection;
00493                 
00494                 // Write some random data to wake up the server.
00495                 channel.writeRaw("x", 1);
00496                 
00497                 clientConnection = channel.readFileDescriptor();
00498                 return ptr(new Client(clientConnection));
00499         }
00500         
00501         /**
00502          * Detach the server, thereby telling it that we don't want to connect
00503          * to it anymore. This frees up some resources in the current process,
00504          * such as file descriptors.
00505          *
00506          * This method is particularily useful to Apache worker processes that
00507          * have just established a connection with the ApplicationPool server.
00508          * Any sessions that are opened prior to calling detach(), will keep
00509          * working even after a detach().
00510          *
00511          * This method may only be called once. The ApplicationPoolServer object
00512          * will become unusable once detach() has been called, so call connect()
00513          * before calling detach().
00514          */
00515         void detach() {
00516                 close(serverSocket);
00517                 serverSocket = -1;
00518         }
00519 };
00520 
00521 typedef shared_ptr<ApplicationPoolServer> ApplicationPoolServerPtr;
00522 
00523 } // namespace Passenger
00524 
00525 #endif /* _PASSENGER_APPLICATION_POOL_SERVER_H_ */

Generated on Mon Apr 28 11:08:59 2008 for Passenger by  doxygen 1.5.3