Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:sschapiro:openstack:upstream
qpid-qmf
mrg.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File mrg.patch of Package qpid-qmf
diff --git a/qpid/cpp/bindings/qmf/python/Makefile.am b/qpid/cpp/bindings/qmf/python/Makefile.am index 421590f..8abad32 100644 --- a/qpid/cpp/bindings/qmf/python/Makefile.am +++ b/qpid/cpp/bindings/qmf/python/Makefile.am @@ -30,11 +30,13 @@ BUILT_SOURCES = $(generated_file_list) SWIG_FLAGS = -w362,401 $(generated_file_list): $(srcdir)/python.i $(srcdir)/../qmfengine.i - swig -c++ -python $(SWIG_FLAGS) $(INCLUDES) $(QPID_CXXFLAGS) -I$(top_srcdir)/src/qmf -I/usr/include -o qmfengine.cpp $(srcdir)/python.i + $(SWIG) -c++ -python $(SWIG_FLAGS) $(INCLUDES) $(QPID_CXXFLAGS) -I$(top_srcdir)/src/qmf -I/usr/include -o qmfengine.cpp $(srcdir)/python.i pylibdir = $(PYTHON_LIB) lib_LTLIBRARIES = _qmfengine.la +qenginedir = $(pyexecdir) +qengine_PYTHON = qmfengine.py qmf.py #_qmfengine_la_LDFLAGS = -avoid-version -module -shrext "$(PYTHON_SO)" #_qmfengine_la_LDFLAGS = -avoid-version -module -shrext ".so" diff --git a/qpid/cpp/bindings/qmf/ruby/Makefile.am b/qpid/cpp/bindings/qmf/ruby/Makefile.am index cfb3a33..de8c4d1 100644 --- a/qpid/cpp/bindings/qmf/ruby/Makefile.am +++ b/qpid/cpp/bindings/qmf/ruby/Makefile.am @@ -35,9 +35,9 @@ qmfengine.cpp: $(srcdir)/ruby.i $(srcdir)/../qmfengine.i rubylibarchdir = $(RUBY_LIB_ARCH) rubylibarch_LTLIBRARIES = qmfengine.la -qmfengine_la_LDFLAGS = -avoid-version -module -shrext ".$(RUBY_DLEXT)" +qmfengine_la_LDFLAGS = -avoid-version -module -shared -shrext ".$(RUBY_DLEXT)" qmfengine_la_LIBADD = $(RUBY_LIBS) -L$(top_builddir)/src/.libs -lqpidclient $(top_builddir)/src/libqmfengine.la -qmfengine_la_CXXFLAGS = $(INCLUDES) -I$(RUBY_INC) -I$(RUBY_INC_ARCH) +qmfengine_la_CXXFLAGS = $(INCLUDES) -I$(RUBY_INC) -I$(RUBY_INC_ARCH) -fno-strict-aliasing nodist_qmfengine_la_SOURCES = qmfengine.cpp CLEANFILES = qmfengine.cpp diff --git a/qpid/cpp/bindings/qmf2/python/Makefile.am b/qpid/cpp/bindings/qmf2/python/Makefile.am index 7adc62e..3dc04e8 100644 --- a/qpid/cpp/bindings/qmf2/python/Makefile.am +++ b/qpid/cpp/bindings/qmf2/python/Makefile.am @@ -30,12 +30,12 @@ BUILT_SOURCES = $(generated_file_list) SWIG_FLAGS = -w362,401 $(generated_file_list): $(srcdir)/python.i $(srcdir)/../qmf2.i $(srcdir)/../../swig_python_typemaps.i - swig -c++ -python $(SWIG_FLAGS) $(INCLUDES) $(QPID_CXXFLAGS) -I/usr/include -o cqmf2.cpp $(srcdir)/python.i + $(SWIG) -c++ -python $(SWIG_FLAGS) $(INCLUDES) $(QPID_CXXFLAGS) -I/usr/include -o cqmf2.cpp $(srcdir)/python.i pylibdir = $(PYTHON_LIB) lib_LTLIBRARIES = _cqmf2.la -cqpiddir = $(pythondir) +cqpiddir = $(pyexecdir) cqpid_PYTHON = qmf2.py cqmf2.py _cqmf2_la_LDFLAGS = -avoid-version -module -shared diff --git a/qpid/cpp/bindings/qmf2/ruby/Makefile.am b/qpid/cpp/bindings/qmf2/ruby/Makefile.am index ae840f8..97bbc6f 100644 --- a/qpid/cpp/bindings/qmf2/ruby/Makefile.am +++ b/qpid/cpp/bindings/qmf2/ruby/Makefile.am @@ -34,9 +34,9 @@ rubylibarchdir = $(RUBY_LIB_ARCH) rubylibarch_LTLIBRARIES = cqmf2.la dist_rubylib_DATA = qmf2.rb -cqmf2_la_LDFLAGS = -avoid-version -module -shrext ".$(RUBY_DLEXT)" +cqmf2_la_LDFLAGS = -avoid-version -module -shared -shrext ".$(RUBY_DLEXT)" cqmf2_la_LIBADD = $(RUBY_LIBS) -L$(top_builddir)/src/.libs -lqmf2 $(top_builddir)/src/libqmf2.la -cqmf2_la_CXXFLAGS = $(INCLUDES) -I$(RUBY_INC) -I$(RUBY_INC_ARCH) +cqmf2_la_CXXFLAGS = $(INCLUDES) -I$(RUBY_INC) -I$(RUBY_INC_ARCH) -fno-strict-aliasing nodist_cqmf2_la_SOURCES = cqmf2.cpp CLEANFILES = cqmf2.cpp diff --git a/qpid/cpp/bindings/qpid/perl/CMakeLists.txt b/qpid/cpp/bindings/qpid/perl/CMakeLists.txt index 92ec534..6edaf28 100644 --- a/qpid/cpp/bindings/qpid/perl/CMakeLists.txt +++ b/qpid/cpp/bindings/qpid/perl/CMakeLists.txt @@ -26,7 +26,7 @@ set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/perl.i PROPERTIES SWIG_F swig_add_module(cqpid_perl perl ${CMAKE_CURRENT_SOURCE_DIR}/perl.i) swig_link_libraries(cqpid_perl qpidmessaging qpidtypes qmf2 ${PERL_LIBRARY}) -set_source_files_properties(${swig_generated_file_fullname} PROPERTIES COMPILE_FLAGS "-I${PERL_INCLUDE_PATH} -I${qpid-cpp_SOURCE_DIR}/include") +set_source_files_properties(${swig_generated_file_fullname} PROPERTIES COMPILE_FLAGS "-fno-strict-aliasing -I${PERL_INCLUDE_PATH} -I${qpid-cpp_SOURCE_DIR}/include") ##---------------------------------- ## Install the complete Perl binding diff --git a/qpid/cpp/bindings/qpid/perl/Makefile.am b/qpid/cpp/bindings/qpid/perl/Makefile.am index 9e47786..da08289 100644 --- a/qpid/cpp/bindings/qpid/perl/Makefile.am +++ b/qpid/cpp/bindings/qpid/perl/Makefile.am @@ -34,7 +34,7 @@ cqpid_perl_PERL = cqpid_perl.pm libcqpid_perl_la_LDFLAGS = -avoid-version -shared libcqpid_perl_la_LIBADD = -L$(top_builddir)/src/.libs -lqpidmessaging -lqpidtypes \ $(top_builddir)/src/libqpidmessaging.la $(top_builddir)/src/libqpidtypes.la -libcqpid_perl_la_CXXFLAGS = $(INCLUDES) +libcqpid_perl_la_CXXFLAGS = $(INCLUDES) -fno-strict-aliasing nodist_libcqpid_perl_la_SOURCES = cqpid_perl.cpp CLEANFILES = cqpid_perl.cpp cqpid_perl.pm diff --git a/qpid/cpp/bindings/qpid/python/Makefile.am b/qpid/cpp/bindings/qpid/python/Makefile.am index 9aef179..dd25f34 100644 --- a/qpid/cpp/bindings/qpid/python/Makefile.am +++ b/qpid/cpp/bindings/qpid/python/Makefile.am @@ -30,12 +30,12 @@ BUILT_SOURCES = $(generated_file_list) SWIG_FLAGS = -w362,401 $(generated_file_list): $(srcdir)/python.i $(srcdir)/../qpid.i $(srcdir)/../../swig_python_typemaps.i - swig -c++ -python $(SWIG_FLAGS) $(INCLUDES) $(QPID_CXXFLAGS) -I$(top_srcdir)/src/qmf -I/usr/include -o cqpid.cpp $(srcdir)/python.i + $(SWIG) -c++ -python $(SWIG_FLAGS) $(INCLUDES) $(QPID_CXXFLAGS) -I$(top_srcdir)/src/qmf -I/usr/include -o cqpid.cpp $(srcdir)/python.i pylibdir = $(PYTHON_LIB) lib_LTLIBRARIES = _cqpid.la -cqpiddir = $(pythondir) +cqpiddir = $(pyexecdir) cqpid_PYTHON = cqpid.py _cqpid_la_LDFLAGS = -avoid-version -module -shared diff --git a/qpid/cpp/bindings/qpid/ruby/CMakeLists.txt b/qpid/cpp/bindings/qpid/ruby/CMakeLists.txt index d403f3f..96c0015 100644 --- a/qpid/cpp/bindings/qpid/ruby/CMakeLists.txt +++ b/qpid/cpp/bindings/qpid/ruby/CMakeLists.txt @@ -26,7 +26,7 @@ set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/ruby.i PROPERTIES SWIG_F swig_add_module(cqpid ruby ${CMAKE_CURRENT_SOURCE_DIR}/ruby.i) swig_link_libraries(cqpid qpidmessaging qpidtypes qmf2 ${RUBY_LIBRARY}) -set_source_files_properties(${swig_generated_file_fullname} PROPERTIES COMPILE_FLAGS "-I${RUBY_INCLUDE_DIR} -I${qpid-cpp_SOURCE_DIR}/include") +set_source_files_properties(${swig_generated_file_fullname} PROPERTIES COMPILE_FLAGS "-fno-strict-aliasing -I${RUBY_INCLUDE_DIR} -I${qpid-cpp_SOURCE_DIR}/include") ##---------------------------------- ## Install the complete Ruby binding diff --git a/qpid/cpp/bindings/qpid/ruby/Makefile.am b/qpid/cpp/bindings/qpid/ruby/Makefile.am index 34f9990..a2a5dd7 100644 --- a/qpid/cpp/bindings/qpid/ruby/Makefile.am +++ b/qpid/cpp/bindings/qpid/ruby/Makefile.am @@ -33,10 +33,10 @@ cqpid.cpp: $(srcdir)/ruby.i $(srcdir)/../qpid.i $(srcdir)/../../swig_ruby_typema rubylibarchdir = $(RUBY_LIB_ARCH) rubylibarch_LTLIBRARIES = cqpid.la -cqpid_la_LDFLAGS = -avoid-version -module -shrext ".$(RUBY_DLEXT)" +cqpid_la_LDFLAGS = -avoid-version -module -shared -shrext ".$(RUBY_DLEXT)" cqpid_la_LIBADD = $(RUBY_LIBS) -L$(top_builddir)/src/.libs -lqpidmessaging -lqpidtypes \ $(top_builddir)/src/libqpidmessaging.la $(top_builddir)/src/libqpidtypes.la -cqpid_la_CXXFLAGS = $(INCLUDES) -I$(RUBY_INC) -I$(RUBY_INC_ARCH) +cqpid_la_CXXFLAGS = $(INCLUDES) -I$(RUBY_INC) -I$(RUBY_INC_ARCH) -fno-strict-aliasing nodist_cqpid_la_SOURCES = cqpid.cpp CLEANFILES = cqpid.cpp diff --git a/qpid/cpp/docs/api/footer.html b/qpid/cpp/docs/api/footer.html index 883410c..5a31e81 100644 --- a/qpid/cpp/docs/api/footer.html +++ b/qpid/cpp/docs/api/footer.html @@ -25,7 +25,7 @@ Qpid C++ API Reference</small></address> <address style="text-align: right;"> <small> -Generated on $datetime for $projectname by <a href="http://www.doxygen.org/index.html"><img src="doxygen.png" alt="doxygen" align="middle" border="0"></a> $doxygenversion</small> +Generated on $date for $projectname by <a href="http://www.doxygen.org/index.html"><img src="doxygen.png" alt="doxygen" align="middle" border="0"></a> $doxygenversion</small> </address> </body> </html> diff --git a/qpid/cpp/docs/man/Makefile.am b/qpid/cpp/docs/man/Makefile.am index 14295f7..b821568 100644 --- a/qpid/cpp/docs/man/Makefile.am +++ b/qpid/cpp/docs/man/Makefile.am @@ -16,10 +16,29 @@ # specific language governing permissions and limitations # under the License. # + +# Generate makefile from qpidd --help +# +# Note: qiddd.1 is normally a _checked in_ pre-generated file, so that +# make dist does not have to build the entire source just for the man page. +# +# To update the checked-in file (e.g. for a new release) do the following: +# +# - start with a completely clean checkout. +# - make sure there are no modules installed in your configured prefix, +# we don't want to pick up configuration from optional modules +# - do bootstrap; configure +# - in build-dir: cd src; make # build the broker +# - in source-dir: cd docs/man; rm qpidd.1 # remove checked-in man page. +# - in build-dir: cd docs/man; make # make new man page +# - edit qpidd.1 to remove all default values referring to file/directory locations. +# these values will differ between builds depending on configuration. +# - if source-dir != build-dir: copy qpidd.1 from build-dir/docs/man to source-dir/docs/man + dist_man_MANS = qpidd.1 -man_aux = $(dist_man_MANS:.1=.x) -EXTRA_DIST = $(man_aux) generate_manpage groffify_options.sed groffify_template.sed +man_aux = $(dist_man_MANS:.1=.x) +EXTRA_DIST = $(man_aux) generate_manpage groffify_options.sed groffify_template.sed DISTCLEANFILES = $(dist_man_MANS) CLEANFILES=qpidd.1 diff --git a/qpid/cpp/docs/man/qpidd.1 b/qpid/cpp/docs/man/qpidd.1 new file mode 100644 index 0000000..48b73b3 --- /dev/null +++ b/qpid/cpp/docs/man/qpidd.1 @@ -0,0 +1,292 @@ +.TH QPIDD "1" "March 2011" "qpidd (qpidc) version 0.11" "User Commands" +.SH NAME + +qpidd \- the Qpid AMQP Message Broker Daemon + +.SH SYNOPSIS + +qpidd [-p port] [--config config_file] [--data-dir directory] + +.SH DESCRIPTION + +An AMQP message broker daemon that stores, routes and forwards +messages using the Advanced Message Queueing Protocol (AMQP). + +.SH OPTIONS + +The options below are built-in to qpidd. Installing add-on modules provides additional options. To see the full set of options available type "qpidd --help" + +Options may be specified via command line, environment variable or configuration file. See FILES and ENVIRONMENT below for details. + +.PP + +.SS Options + +.TP +\-h [ \-\-help ] +Displays the +help message +.TP +\-v [ \-\-version ] +Displays version +information +.TP +\-\-config FILE +Reads configurat +ion from FILE + +.SS Module options +.TP +\-\-module\-dir DIR +Load all +shareable +modules in +this directo +ry +.TP +\-\-load\-module FILE +Specifies +additional +module(s) to +be loaded +.TP +\-\-no\-module\-dir +Don't load +modules from +module +directory + +.SS Broker Options +.TP +\-\-data\-dir DIR +Directory to contain +persistent data generated +by the broker +.TP +\-\-no\-data\-dir +Don't use a data +directory. No persistent +configuration will be +loaded or stored +.TP +\-p [ \-\-port ] PORT (5672) +Tells the broker to +listen on PORT +.TP +\-\-worker\-threads N (9) +Sets the broker thread +pool size +.TP +\-\-max\-connections N (500) +Sets the maximum allowed +connections +.TP +\-\-connection\-backlog N (10) +Sets the connection +backlog limit for the +server socket +.TP +\-m [ \-\-mgmt\-enable ] yes|no (1) +Enable Management +.TP +\-\-mgmt\-qmf2 yes|no (1) +Enable broadcast of +management information +over QMF v2 +.TP +\-\-mgmt\-qmf1 yes|no (1) +Enable broadcast of +management information +over QMF v1 +.TP +\-\-mgmt\-pub\-interval SECONDS (10) +Management Publish +Interval +.TP +\-\-queue\-purge\-interval SECONDS (600) +Interval between attempts +to purge any expired +messages from queues +.TP +\-\-auth yes|no (1) +Enable authentication, if +disabled all incoming +connections will be +trusted +.TP +\-\-realm REALM (QPID) +Use the given realm when +performing authentication +.TP +\-\-default\-queue\-limit BYTES (104857600) +Default maximum size for +queues (in bytes) +.TP +\-\-tcp\-nodelay +Set TCP_NODELAY on TCP +connections +.TP +\-\-require\-encryption +Only accept connections +that are encrypted +\-\-known\-hosts\-url URL or \&'none' URL to send as +\&'known\-hosts' to clients +(\&'none' implies empty +list) +.TP +\-\-sasl\-config DIR +gets sasl config info +from nonstandard location +.TP +\-\-max\-session\-rate MESSAGES/S (0) +Sets the maximum message +rate per session +(0=unlimited) +.TP +\-\-async\-queue\-events yes|no (0) +Set Queue Events async, +used for services like +replication +.TP +\-\-default\-flow\-stop\-threshold %MESSAGES (80) +Queue capacity level at +which flow control is +activated. +.TP +\-\-default\-flow\-resume\-threshold %MESSAGES (70) +Queue capacity level at +which flow control is +de\-activated. +.TP +\-\-default\-event\-threshold\-ratio %age of limit (80) +The ratio of any +specified queue limit at +which an event will be +raised + +.SS Logging options +.TP +\-t [ \-\-trace ] +Enables all logging +.TP +\-\-log\-enable RULE (notice+) +Enables logging for selected levels +and components. RULE is in the form +\&'LEVEL[+][:PATTERN]' Levels are one +of: +trace debug info notice warning +error critical +For example: +\&'\-\-log\-enable warning+' logs all +warning, error and critical messages. +\&'\-\-log\-enable debug:framing' logs +debug messages from the framing +namespace. This option can be used +multiple times +.TP +\-\-log\-time yes|no (1) +Include time in log messages +.TP +\-\-log\-level yes|no (1) +Include severity level in log +messages +.TP +\-\-log\-source yes|no (0) +Include source file:line in log +messages +.TP +\-\-log\-thread yes|no (0) +Include thread ID in log messages +.TP +\-\-log\-function yes|no (0) +Include function signature in log +messages +.TP +\-\-log\-prefix STRING +Prefix to append to all log messages + +.SS Logging sink options +.TP +\-\-log\-to\-stderr yes|no (1) +Send logging output to stderr +.TP +\-\-log\-to\-stdout yes|no (0) +Send logging output to stdout +.TP +\-\-log\-to\-file FILE +Send log output to FILE. +.TP +\-\-log\-to\-syslog yes|no (0) +Send logging output to syslog; +customize using \-\-syslog\-name and +\-\-syslog\-facility +.TP +\-\-syslog\-name NAME (lt\-qpidd) +Name to use in syslog messages +.TP +\-\-syslog\-facility LOG_XXX (LOG_DAEMON) +Facility to use in syslog messages + +.SS Daemon options +.TP +\-d [ \-\-daemon ] +Run as a daemon. Logs to syslog +by default in this mode. +.TP +\-\-transport TRANSPORT (tcp) +The transport for which to +return the port +.TP +\-\-pid\-dir DIR +Directory where port\-specific +PID file is stored +.TP +\-w [ \-\-wait ] SECONDS (600) +Sets the maximum wait time to +initialize the daemon. If the +daemon fails to initialize, +prints an error and returns 1 +.TP +\-c [ \-\-check ] +Prints the daemon's process ID +to stdout and returns 0 if the +daemon is running, otherwise +returns 1 +.TP +\-q [ \-\-quit ] +Tells the daemon to shut down + +.SH FILES +.I /etc/qpidd.conf +.RS +Default configuration file. +.RE + +Configuration file settings are over-ridden by command line or environment variable settings. '--config <file>' or 'export QPID_CONFIG=<file>' specifies an alternate file. + +Each line is a name=value pair. Blank lines and lines beginning with # are ignored. For example: + + # My qpidd configuration file. + port=6000 + max-connections=10 + log-to-file=/tmp/qpidd.log + +.SH ENVIRONMENT +.I QPID_<option> +.RS +There is an environment variable for each option. +.RE + +The environment variable is the option name in uppercase, prefixed with QPID_ and '.' or '-' are replaced with '_'. Environment settings are over-ridden by command line settings. For example: + + export QPID_PORT=6000 + export QPID_MAX_CONNECTIONS=10 + export QPID_LOG_TO_FILE=/tmp/qpidd.log + +.SH AUTHOR + +The Apache Qpid Project, dev@qpid.apache.org + +.SH REPORTING BUGS + +Please report bugs to users@qpid.apache.org diff --git a/qpid/cpp/docs/man/qpidd.x b/qpid/cpp/docs/man/qpidd.x index af5d962..0ccf3b5 100644 --- a/qpid/cpp/docs/man/qpidd.x +++ b/qpid/cpp/docs/man/qpidd.x @@ -13,6 +13,8 @@ messages using the Advanced Message Queueing Protocol (AMQP). [OPTIONS] +The options below are built-in to qpidd. Installing add-on modules provides additional options. To see the full set of options available type "qpidd --help" + Options may be specified via command line, environment variable or configuration file. See FILES and ENVIRONMENT below for details. [FILES] diff --git a/qpid/cpp/examples/old_api/Makefile.am b/qpid/cpp/examples/old_api/Makefile.am index 466eee2..04216ff 100644 --- a/qpid/cpp/examples/old_api/Makefile.am +++ b/qpid/cpp/examples/old_api/Makefile.am @@ -36,7 +36,7 @@ $(MAKEDIST): Makefile examplesdir=$(pkgdatadir)/examples/old_api dist_examples_DATA = $(MAKEDIST) -EXTRA_DIST = README.verify verify verify_all +EXTRA_DIST = README.verify verify verify_all CMakeLists.txt # For older versions of automake abs_top_srcdir = @abs_top_srcdir@ diff --git a/qpid/cpp/include/qpid/sys/ExceptionHolder.h b/qpid/cpp/include/qpid/sys/ExceptionHolder.h index 9eff1d6..4bc934c 100644 --- a/qpid/cpp/include/qpid/sys/ExceptionHolder.h +++ b/qpid/cpp/include/qpid/sys/ExceptionHolder.h @@ -10,9 +10,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -42,14 +42,11 @@ class ExceptionHolder : public Raisable { public: ExceptionHolder() {} // Use default copy & assign. - + /** Take ownership of ex */ template <class Ex> ExceptionHolder(Ex* ex) { wrap(ex); } - template <class Ex> ExceptionHolder(const boost::shared_ptr<Ex>& ex) { wrap(ex.release()); } - template <class Ex> ExceptionHolder& operator=(Ex* ex) { wrap(ex); return *this; } - template <class Ex> ExceptionHolder& operator=(boost::shared_ptr<Ex> ex) { wrap(ex.release()); return *this; } - + void raise() const { if (wrapper.get()) wrapper->raise() ; } std::string what() const { return wrapper.get() ? wrapper->what() : std::string(); } bool empty() const { return !wrapper.get(); } @@ -66,7 +63,7 @@ class ExceptionHolder : public Raisable { template <class Ex> void wrap(Ex* ex) { wrapper.reset(new Wrapper<Ex>(ex)); } boost::shared_ptr<Raisable> wrapper; }; - + }} // namespace qpid::sys diff --git a/qpid/cpp/include/qpid/types/Variant.h b/qpid/cpp/include/qpid/types/Variant.h index d926013..876e1a2 100644 --- a/qpid/cpp/include/qpid/types/Variant.h +++ b/qpid/cpp/include/qpid/types/Variant.h @@ -120,6 +120,10 @@ class Variant * value. Recognises integers, doubles and booleans. */ QPID_TYPES_EXTERN Variant& parse(const std::string&); + /** + * fromString() is deprecated - use parse() instead. + */ + QPID_TYPES_EXTERN Variant& fromString(const std::string& s); QPID_TYPES_EXTERN bool asBool() const; QPID_TYPES_EXTERN uint8_t asUint8() const; diff --git a/qpid/cpp/managementgen/Makefile.am b/qpid/cpp/managementgen/Makefile.am index 6c2024c..e10dd63 100644 --- a/qpid/cpp/managementgen/Makefile.am +++ b/qpid/cpp/managementgen/Makefile.am @@ -19,10 +19,16 @@ qmfpythondir = $(pythondir) dist_bin_SCRIPTS = \ qmf-gen -nobase_qmfpython_DATA = \ + +pkgpyexec_qmfgendir = $(pyexecdir)/qmfgen +pkgpyexec_qmfgen_PYTHON = \ qmfgen/__init__.py \ qmfgen/generate.py \ qmfgen/schema.py \ + qmfgen/management-types.xml + +pkgpyexec_qmfgentmpldir = $(pyexecdir)/qmfgen/templates +pkgpyexec_qmfgentmpl_PYTHON = \ qmfgen/templates/Args.h \ qmfgen/templates/Class.cpp \ qmfgen/templates/Class.h \ @@ -32,7 +38,6 @@ nobase_qmfpython_DATA = \ qmfgen/templates/Package.cpp \ qmfgen/templates/Package.h \ qmfgen/templates/V2Package.cpp \ - qmfgen/templates/V2Package.h \ - qmfgen/management-types.xml + qmfgen/templates/V2Package.h EXTRA_DIST = $(nobase_qmfpython_DATA) CMakeLists.txt diff --git a/qpid/cpp/managementgen/qmfgen/schema.py b/qpid/cpp/managementgen/qmfgen/schema.py index afdfe42..59e951f 100755 --- a/qpid/cpp/managementgen/qmfgen/schema.py +++ b/qpid/cpp/managementgen/qmfgen/schema.py @@ -1731,9 +1731,9 @@ class SchemaPackage: stream.write(" qmf::SchemaProperty arg(\"%s\", %s);\n" % (arg.name, typeName)) if subType: stream.write(" arg.setSubtype(\"%s\");\n" % subType) - if stat.unit: + if arg.unit: stream.write(" arg.setUnit(\"%s\");\n" % arg.unit) - if stat.desc: + if arg.desc: stream.write(" arg.setDesc(\"%s\");\n" % arg.desc) stream.write(" arg.setDirection(%s);\n" % self.qmfv2Dir(arg.dir)) stream.write(" method.addArgument(arg);\n") diff --git a/qpid/cpp/rubygen/amqpgen.rb b/qpid/cpp/rubygen/amqpgen.rb index 69e65a4..20aac35 100755 --- a/qpid/cpp/rubygen/amqpgen.rb +++ b/qpid/cpp/rubygen/amqpgen.rb @@ -61,7 +61,8 @@ end class Module # Add trailing _ to avoid conflict with Object methods. def mangle(sym) - (Object.method_defined? sym) ? (sym.to_s+"_").intern : sym + sym = (sym.to_s+"_").to_sym if (Object.method_defined?(sym) or sym == :type) + sym end # Add attribute reader for XML attribute. diff --git a/qpid/cpp/src/Makefile.am b/qpid/cpp/src/Makefile.am index dfb2547..9eaa76a 100644 --- a/qpid/cpp/src/Makefile.am +++ b/qpid/cpp/src/Makefile.am @@ -332,6 +332,7 @@ libqpidcommon_la_SOURCES += \ qpid/Address.cpp \ qpid/DataDir.cpp \ qpid/DataDir.h \ + qpid/DisableExceptionLogging.h \ qpid/Exception.cpp \ qpid/Modules.cpp \ qpid/Modules.h \ @@ -740,7 +741,7 @@ libqpidclient_la_SOURCES = \ QPIDCLIENT_VERSION_INFO = 2:0:0 libqpidclient_la_LDFLAGS = -version-info $(QPIDCLIENT_VERSION_INFO) -libqpidtypes_la_libadd=-luuid +libqpidtypes_la_LIBADD= -luuid libqpidtypes_la_SOURCES= \ qpid/types/Exception.cpp \ qpid/types/Uuid.cpp \ @@ -884,10 +885,6 @@ nobase_include_HEADERS += \ ../include/qpid/types/Variant.h \ ../include/qpid/types/ImportExport.h -# Force build of qpidd during dist phase so help2man will work. -dist-hook: $(BUILT_SOURCES) - $(MAKE) qpidd - # Create the default data directory install-data-local: $(mkinstalldirs) $(DESTDIR)/$(localstatedir)/lib/qpidd diff --git a/qpid/cpp/src/qpid/DisableExceptionLogging.h b/qpid/cpp/src/qpid/DisableExceptionLogging.h new file mode 100644 index 0000000..04a9240 --- /dev/null +++ b/qpid/cpp/src/qpid/DisableExceptionLogging.h @@ -0,0 +1,39 @@ +#ifndef QPID_DISABLEEXCEPTIONLOGGING_H +#define QPID_DISABLEEXCEPTIONLOGGING_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/CommonImportExport.h" + +namespace qpid { + +/** + * Temporarily disable logging in qpid::Exception constructor. + * Used by log::Logger to avoid logging exceptions during Logger construction. + */ +struct DisableExceptionLogging +{ + QPID_COMMON_EXTERN DisableExceptionLogging(); + QPID_COMMON_EXTERN ~DisableExceptionLogging(); +}; +} // namespace qpid + +#endif /*!QPID_DISABLEEXCEPTIONLOGGING_H*/ diff --git a/qpid/cpp/src/qpid/Exception.cpp b/qpid/cpp/src/qpid/Exception.cpp index 16a3a13..a6696f0 100644 --- a/qpid/cpp/src/qpid/Exception.cpp +++ b/qpid/cpp/src/qpid/Exception.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,13 +21,25 @@ #include "qpid/log/Statement.h" #include "qpid/Exception.h" +#include "qpid/DisableExceptionLogging.h" #include <typeinfo> #include <assert.h> #include <string.h> namespace qpid { +// Note on static initialization order: if an exception is constructed +// in a static constructor before disableExceptionLogging has been +// initialized, the worst that can happen is we lose an exception log +// message. Since we shouldn't be throwing a lot of exceptions during +// static construction this seems safe. +static bool disableExceptionLogging = false; + +DisableExceptionLogging::DisableExceptionLogging() { disableExceptionLogging = true; } +DisableExceptionLogging::~DisableExceptionLogging() { disableExceptionLogging = false; } + Exception::Exception(const std::string& msg) throw() : message(msg) { + if (disableExceptionLogging) return; QPID_LOG_IF(debug, !msg.empty(), "Exception constructed: " << message); } diff --git a/qpid/cpp/src/qpid/SaslFactory.cpp b/qpid/cpp/src/qpid/SaslFactory.cpp index 055883a..f117404 100644 --- a/qpid/cpp/src/qpid/SaslFactory.cpp +++ b/qpid/cpp/src/qpid/SaslFactory.cpp @@ -182,17 +182,18 @@ CyrusSasl::CyrusSasl(const std::string & username, const std::string & password, callbacks[i].id = SASL_CB_AUTHNAME; callbacks[i].proc = (CallbackProc*) &getUserFromSettings; callbacks[i++].context = &settings; - } - callbacks[i].id = SASL_CB_PASS; - if (settings.password.empty()) { - callbacks[i].proc = 0; - callbacks[i++].context = 0; - } else { - callbacks[i].proc = (CallbackProc*) &getPasswordFromSettings; - callbacks[i++].context = &settings; + callbacks[i].id = SASL_CB_PASS; + if (settings.password.empty()) { + callbacks[i].proc = 0; + callbacks[i++].context = 0; + } else { + callbacks[i].proc = (CallbackProc*) &getPasswordFromSettings; + callbacks[i++].context = &settings; + } } + callbacks[i].id = SASL_CB_LIST_END; callbacks[i].proc = 0; callbacks[i++].context = 0; diff --git a/qpid/cpp/src/qpid/agent/ManagementAgentImpl.cpp b/qpid/cpp/src/qpid/agent/ManagementAgentImpl.cpp index 593d403..633401e 100644 --- a/qpid/cpp/src/qpid/agent/ManagementAgentImpl.cpp +++ b/qpid/cpp/src/qpid/agent/ManagementAgentImpl.cpp @@ -305,43 +305,47 @@ void ManagementAgentImpl::raiseEvent(const ManagementEvent& event, severity_t se "emerg", "alert", "crit", "error", "warn", "note", "info", "debug" }; - sys::Mutex::ScopedLock lock(agentLock); - Buffer outBuffer(eventBuffer, MA_BUFFER_SIZE); - uint8_t sev = (severity == SEV_DEFAULT) ? event.getSeverity() : (uint8_t) severity; + string content; stringstream key; - - // key << "console.event." << assignedBrokerBank << "." << assignedAgentBank << "." << - // event.getPackageName() << "." << event.getEventName(); - key << "agent.ind.event." << keyifyNameStr(event.getPackageName()) - << "." << keyifyNameStr(event.getEventName()) - << "." << severityStr[sev] - << "." << vendorNameKey - << "." << productNameKey - << "." << instanceNameKey; - - Variant::Map map_; - Variant::Map schemaId; - Variant::Map values; Variant::Map headers; - string content; - map_["_schema_id"] = mapEncodeSchemaId(event.getPackageName(), - event.getEventName(), - event.getMd5Sum(), - ManagementItem::CLASS_KIND_EVENT); - event.mapEncode(values); - map_["_values"] = values; - map_["_timestamp"] = uint64_t(Duration(EPOCH, now())); - map_["_severity"] = sev; + { + sys::Mutex::ScopedLock lock(agentLock); + Buffer outBuffer(eventBuffer, MA_BUFFER_SIZE); + uint8_t sev = (severity == SEV_DEFAULT) ? event.getSeverity() : (uint8_t) severity; + + // key << "console.event." << assignedBrokerBank << "." << assignedAgentBank << "." << + // event.getPackageName() << "." << event.getEventName(); + key << "agent.ind.event." << keyifyNameStr(event.getPackageName()) + << "." << keyifyNameStr(event.getEventName()) + << "." << severityStr[sev] + << "." << vendorNameKey + << "." << productNameKey + << "." << instanceNameKey; + + Variant::Map map_; + Variant::Map schemaId; + Variant::Map values; + + map_["_schema_id"] = mapEncodeSchemaId(event.getPackageName(), + event.getEventName(), + event.getMd5Sum(), + ManagementItem::CLASS_KIND_EVENT); + event.mapEncode(values); + map_["_values"] = values; + map_["_timestamp"] = uint64_t(Duration(EPOCH, now())); + map_["_severity"] = sev; + + headers["method"] = "indication"; + headers["qmf.opcode"] = "_data_indication"; + headers["qmf.content"] = "_event"; + headers["qmf.agent"] = name_address; - headers["method"] = "indication"; - headers["qmf.opcode"] = "_data_indication"; - headers["qmf.content"] = "_event"; - headers["qmf.agent"] = name_address; + Variant::List list; + list.push_back(map_); + ListCodec::encode(list, content); + } - Variant::List list; - list.push_back(map_); - ListCodec::encode(list, content); connThreadBody.sendBuffer(content, "", headers, topicExchange, key.str(), "amqp/list"); } @@ -521,9 +525,12 @@ void ManagementAgentImpl::sendException(const string& rte, const string& rtk, co void ManagementAgentImpl::handleSchemaRequest(Buffer& inBuffer, uint32_t sequence, const string& rte, const string& rtk) { - sys::Mutex::ScopedLock lock(agentLock); string packageName; SchemaClassKey key; + uint32_t outLen(0); + char localBuffer[MA_BUFFER_SIZE]; + Buffer outBuffer(localBuffer, MA_BUFFER_SIZE); + bool found(false); inBuffer.getShortString(packageName); inBuffer.getShortString(key.name); @@ -531,26 +538,30 @@ void ManagementAgentImpl::handleSchemaRequest(Buffer& inBuffer, uint32_t sequenc QPID_LOG(trace, "RCVD SchemaRequest: package=" << packageName << " class=" << key.name); - PackageMap::iterator pIter = packages.find(packageName); - if (pIter != packages.end()) { - ClassMap& cMap = pIter->second; - ClassMap::iterator cIter = cMap.find(key); - if (cIter != cMap.end()) { - SchemaClass& schema = cIter->second; - Buffer outBuffer(outputBuffer, MA_BUFFER_SIZE); - uint32_t outLen; - string body; - - encodeHeader(outBuffer, 's', sequence); - schema.writeSchemaCall(body); - outBuffer.putRawData(body); - outLen = MA_BUFFER_SIZE - outBuffer.available(); - outBuffer.reset(); - connThreadBody.sendBuffer(outBuffer, outLen, rte, rtk); - - QPID_LOG(trace, "SENT SchemaInd: package=" << packageName << " class=" << key.name); + { + sys::Mutex::ScopedLock lock(agentLock); + PackageMap::iterator pIter = packages.find(packageName); + if (pIter != packages.end()) { + ClassMap& cMap = pIter->second; + ClassMap::iterator cIter = cMap.find(key); + if (cIter != cMap.end()) { + SchemaClass& schema = cIter->second; + string body; + + encodeHeader(outBuffer, 's', sequence); + schema.writeSchemaCall(body); + outBuffer.putRawData(body); + outLen = MA_BUFFER_SIZE - outBuffer.available(); + outBuffer.reset(); + found = true; + } } } + + if (found) { + connThreadBody.sendBuffer(outBuffer, outLen, rte, rtk); + QPID_LOG(trace, "SENT SchemaInd: package=" << packageName << " class=" << key.name); + } } void ManagementAgentImpl::handleConsoleAddedIndication() @@ -969,18 +980,6 @@ ManagementAgentImpl::PackageMap::iterator ManagementAgentImpl::findOrAddPackage( pair<PackageMap::iterator, bool> result = packages.insert(pair<string, ClassMap>(name, ClassMap())); - if (connected) { - // Publish a package-indication message - Buffer outBuffer(outputBuffer, MA_BUFFER_SIZE); - uint32_t outLen; - - encodeHeader(outBuffer, 'p'); - encodePackageIndication(outBuffer, result.first); - outLen = MA_BUFFER_SIZE - outBuffer.available(); - outBuffer.reset(); - connThreadBody.sendBuffer(outBuffer, outLen, "qpid.management", "schema.package"); - } - return result.first; } @@ -1038,131 +1037,146 @@ void ManagementAgentImpl::encodeClassIndication(Buffer& buf, QPID_LOG(trace, "SENT ClassInd: package=" << (*pIter).first << " class=" << key.name); } +struct MessageItem { + string content; + Variant::Map headers; + string key; + MessageItem(const Variant::Map& h, const string& k) : headers(h), key(k) {} +}; + void ManagementAgentImpl::periodicProcessing() { string addr_key_base = "agent.ind.data."; - sys::Mutex::ScopedLock lock(agentLock); list<ObjectId> deleteList; - - if (!connected) - return; + list<boost::shared_ptr<MessageItem> > message_list; sendHeartbeat(); - moveNewObjectsLH(); - - // - // Clear the been-here flag on all objects in the map. - // - for (ObjectMap::iterator iter = managementObjects.begin(); - iter != managementObjects.end(); - iter++) { - ManagementObject* object = iter->second.get(); - object->setFlags(0); - if (publishAllData) { - object->setForcePublish(true); - } - } - - publishAllData = false; + { + sys::Mutex::ScopedLock lock(agentLock); - // - // Process the entire object map. - // - uint32_t v2Objs = 0; + if (!connected) + return; - for (ObjectMap::iterator baseIter = managementObjects.begin(); - baseIter != managementObjects.end(); - baseIter++) { - ManagementObject* baseObject = baseIter->second.get(); + moveNewObjectsLH(); // - // Skip until we find a base object requiring a sent message. + // Clear the been-here flag on all objects in the map. // - if (baseObject->getFlags() == 1 || - (!baseObject->getConfigChanged() && - !baseObject->getInstChanged() && - !baseObject->getForcePublish() && - !baseObject->isDeleted())) - continue; - - std::string packageName = baseObject->getPackageName(); - std::string className = baseObject->getClassName(); - - Variant::List list_; - string content; - std::stringstream addr_key; - Variant::Map headers; - - addr_key << addr_key_base; - addr_key << keyifyNameStr(packageName) - << "." << keyifyNameStr(className) - << "." << vendorNameKey - << "." << productNameKey - << "." << instanceNameKey; - - headers["method"] = "indication"; - headers["qmf.opcode"] = "_data_indication"; - headers["qmf.content"] = "_data"; - headers["qmf.agent"] = name_address; - - for (ObjectMap::iterator iter = baseIter; + for (ObjectMap::iterator iter = managementObjects.begin(); iter != managementObjects.end(); iter++) { ManagementObject* object = iter->second.get(); - bool send_stats, send_props; - if (baseObject->isSameClass(*object) && object->getFlags() == 0) { - object->setFlags(1); - if (object->getConfigChanged() || object->getInstChanged()) - object->setUpdateTime(); + object->setFlags(0); + if (publishAllData) { + object->setForcePublish(true); + } + } - send_props = (object->getConfigChanged() || object->getForcePublish() || object->isDeleted()); - send_stats = (object->hasInst() && (object->getInstChanged() || object->getForcePublish())); - - if (send_stats || send_props) { - Variant::Map map_; - Variant::Map values; - Variant::Map oid; - - object->getObjectId().mapEncode(oid); - map_["_object_id"] = oid; - map_["_schema_id"] = mapEncodeSchemaId(object->getPackageName(), - object->getClassName(), - object->getMd5Sum()); - object->writeTimestamps(map_); - object->mapEncodeValues(values, send_props, send_stats); - map_["_values"] = values; - list_.push_back(map_); - - if (++v2Objs >= maxV2ReplyObjs) { - v2Objs = 0; - ListCodec::encode(list_, content); - - connThreadBody.sendBuffer(content, "", headers, topicExchange, addr_key.str(), "amqp/list"); - list_.clear(); - content.clear(); - QPID_LOG(trace, "SENT DataIndication"); + publishAllData = false; + + // + // Process the entire object map. + // + uint32_t v2Objs = 0; + + for (ObjectMap::iterator baseIter = managementObjects.begin(); + baseIter != managementObjects.end(); + baseIter++) { + ManagementObject* baseObject = baseIter->second.get(); + + // + // Skip until we find a base object requiring a sent message. + // + if (baseObject->getFlags() == 1 || + (!baseObject->getConfigChanged() && + !baseObject->getInstChanged() && + !baseObject->getForcePublish() && + !baseObject->isDeleted())) + continue; + + std::string packageName = baseObject->getPackageName(); + std::string className = baseObject->getClassName(); + + Variant::List list_; + std::stringstream addr_key; + Variant::Map headers; + + addr_key << addr_key_base; + addr_key << keyifyNameStr(packageName) + << "." << keyifyNameStr(className) + << "." << vendorNameKey + << "." << productNameKey + << "." << instanceNameKey; + + headers["method"] = "indication"; + headers["qmf.opcode"] = "_data_indication"; + headers["qmf.content"] = "_data"; + headers["qmf.agent"] = name_address; + + for (ObjectMap::iterator iter = baseIter; + iter != managementObjects.end(); + iter++) { + ManagementObject* object = iter->second.get(); + bool send_stats, send_props; + if (baseObject->isSameClass(*object) && object->getFlags() == 0) { + object->setFlags(1); + if (object->getConfigChanged() || object->getInstChanged()) + object->setUpdateTime(); + + send_props = (object->getConfigChanged() || object->getForcePublish() || object->isDeleted()); + send_stats = (object->hasInst() && (object->getInstChanged() || object->getForcePublish())); + + if (send_stats || send_props) { + Variant::Map map_; + Variant::Map values; + Variant::Map oid; + + object->getObjectId().mapEncode(oid); + map_["_object_id"] = oid; + map_["_schema_id"] = mapEncodeSchemaId(object->getPackageName(), + object->getClassName(), + object->getMd5Sum()); + object->writeTimestamps(map_); + object->mapEncodeValues(values, send_props, send_stats); + map_["_values"] = values; + list_.push_back(map_); + + if (++v2Objs >= maxV2ReplyObjs) { + v2Objs = 0; + boost::shared_ptr<MessageItem> item(new MessageItem(headers, addr_key.str())); + ListCodec::encode(list_, item->content); + message_list.push_back(item); + list_.clear(); + } } + + if (object->isDeleted()) + deleteList.push_back(iter->first); + object->setForcePublish(false); } + } - if (object->isDeleted()) - deleteList.push_back(iter->first); - object->setForcePublish(false); + if (!list_.empty()) { + boost::shared_ptr<MessageItem> item(new MessageItem(headers, addr_key.str())); + ListCodec::encode(list_, item->content); + message_list.push_back(item); } } - if (!list_.empty()) { - ListCodec::encode(list_, content); - connThreadBody.sendBuffer(content, "", headers, topicExchange, addr_key.str(), "amqp/list"); - QPID_LOG(trace, "SENT DataIndication"); - } + // Delete flagged objects + for (list<ObjectId>::reverse_iterator iter = deleteList.rbegin(); + iter != deleteList.rend(); + iter++) + managementObjects.erase(*iter); } - // Delete flagged objects - for (list<ObjectId>::reverse_iterator iter = deleteList.rbegin(); - iter != deleteList.rend(); - iter++) - managementObjects.erase(*iter); + while (!message_list.empty()) { + boost::shared_ptr<MessageItem> item(message_list.front()); + message_list.pop_front(); + connThreadBody.sendBuffer(item->content, "", item->headers, topicExchange, item->key, "amqp/list"); + QPID_LOG(trace, "SENT DataIndication"); + } } diff --git a/qpid/cpp/src/qpid/broker/Broker.cpp b/qpid/cpp/src/qpid/broker/Broker.cpp index b20c346..cb2e152 100644 --- a/qpid/cpp/src/qpid/broker/Broker.cpp +++ b/qpid/cpp/src/qpid/broker/Broker.cpp @@ -248,13 +248,7 @@ Broker::Broker(const Broker::Options& conf) : // Early-Initialize plugins Plugin::earlyInitAll(*this); - /** todo KAG - remove once cluster support for flow control done */ - if (isInCluster()) { - QPID_LOG(info, "Producer Flow Control TBD for clustered brokers - queue flow control disabled by default."); - QueueFlowLimit::setDefaults(0, 0, 0); - } else { - QueueFlowLimit::setDefaults(conf.queueLimit, conf.queueFlowStopRatio, conf.queueFlowResumeRatio); - } + QueueFlowLimit::setDefaults(conf.queueLimit, conf.queueFlowStopRatio, conf.queueFlowResumeRatio); // If no plugin store module registered itself, set up the null store. if (NullMessageStore::isNullStore(store.get())) diff --git a/qpid/cpp/src/qpid/broker/DirectExchange.cpp b/qpid/cpp/src/qpid/broker/DirectExchange.cpp index 5b8104c..060f80f 100644 --- a/qpid/cpp/src/qpid/broker/DirectExchange.cpp +++ b/qpid/cpp/src/qpid/broker/DirectExchange.cpp @@ -94,7 +94,7 @@ bool DirectExchange::bind(Queue::shared_ptr queue, const string& routingKey, con propagate = bk.fedBinding.delOrigin(queue->getName(), fedOrigin); if (bk.fedBinding.countFedBindings(queue->getName()) == 0) - unbind(queue, routingKey, 0); + unbind(queue, routingKey, args); } else if (fedOp == fedOpReorigin) { /** gather up all the keys that need rebinding in a local vector @@ -124,17 +124,18 @@ bool DirectExchange::bind(Queue::shared_ptr queue, const string& routingKey, con return true; } -bool DirectExchange::unbind(Queue::shared_ptr queue, const string& routingKey, const FieldTable* /*args*/) +bool DirectExchange::unbind(Queue::shared_ptr queue, const string& routingKey, const FieldTable* args) { + string fedOrigin(args ? args->getAsString(qpidFedOrigin) : ""); bool propagate = false; - QPID_LOG(debug, "Unbind key [" << routingKey << "] from queue " << queue->getName()); - + QPID_LOG(debug, "Unbinding key [" << routingKey << "] from queue " << queue->getName() + << " on exchange " << getName() << " origin=" << fedOrigin << ")" ); { Mutex::ScopedLock l(lock); BoundKey& bk = bindings[routingKey]; if (bk.queues.remove_if(MatchQueue(queue))) { - propagate = bk.fedBinding.delOrigin(); + propagate = bk.fedBinding.delOrigin(queue->getName(), fedOrigin); if (mgmtExchange != 0) { mgmtExchange->dec_bindingCount(); } diff --git a/qpid/cpp/src/qpid/broker/Exchange.cpp b/qpid/cpp/src/qpid/broker/Exchange.cpp index 2c7589e..622cc81 100644 --- a/qpid/cpp/src/qpid/broker/Exchange.cpp +++ b/qpid/cpp/src/qpid/broker/Exchange.cpp @@ -19,16 +19,18 @@ * */ +#include "qpid/broker/Broker.h" +#include "qpid/broker/DeliverableMessage.h" #include "qpid/broker/Exchange.h" #include "qpid/broker/ExchangeRegistry.h" #include "qpid/broker/FedOps.h" -#include "qpid/broker/Broker.h" -#include "qpid/management/ManagementAgent.h" #include "qpid/broker/Queue.h" -#include "qpid/log/Statement.h" #include "qpid/framing/MessageProperties.h" #include "qpid/framing/reply_exceptions.h" -#include "qpid/broker/DeliverableMessage.h" +#include "qpid/log/Statement.h" +#include "qpid/management/ManagementAgent.h" +#include "qpid/sys/ExceptionHolder.h" +#include <stdexcept> using namespace qpid::broker; using namespace qpid::framing; @@ -70,6 +72,36 @@ Exchange::PreRoute::~PreRoute(){ } } +namespace { +/** Store information about an exception to be thrown later. + * If multiple exceptions are stored, save the first of the "most severe" + * exceptions, SESSION is les sever than CONNECTION etc. + */ +class ExInfo { + public: + enum Type { NONE, SESSION, CONNECTION, OTHER }; + + ExInfo(string exchange) : type(NONE), exchange(exchange) {} + void store(Type type_, const qpid::sys::ExceptionHolder& exception_, const boost::shared_ptr<Queue>& queue) { + QPID_LOG(warning, "Exchange " << exchange << " cannot deliver to queue " + << queue->getName() << ": " << exception_.what()); + if (type < type_) { // Replace less severe exception + type = type_; + exception = exception_; + } + } + + void raise() { + exception.raise(); + } + + private: + Type type; + string exchange; + qpid::sys::ExceptionHolder exception; +}; +} + void Exchange::doRoute(Deliverable& msg, ConstBindingList b) { int count = 0; @@ -80,11 +112,25 @@ void Exchange::doRoute(Deliverable& msg, ConstBindingList b) msg.getMessage().blockContentRelease(); } + + ExInfo error(getName()); // Save exception to throw at the end. for(std::vector<Binding::shared_ptr>::const_iterator i = b->begin(); i != b->end(); i++, count++) { - msg.deliverTo((*i)->queue); - if ((*i)->mgmtBinding != 0) - (*i)->mgmtBinding->inc_msgMatched(); + try { + msg.deliverTo((*i)->queue); + if ((*i)->mgmtBinding != 0) + (*i)->mgmtBinding->inc_msgMatched(); + } + catch (const SessionException& e) { + error.store(ExInfo::SESSION, framing::createSessionException(e.code, e.what()),(*i)->queue); + } + catch (const ConnectionException& e) { + error.store(ExInfo::CONNECTION, framing::createConnectionException(e.code, e.what()), (*i)->queue); + } + catch (const std::exception& e) { + error.store(ExInfo::OTHER, qpid::sys::ExceptionHolder(new Exception(e.what())), (*i)->queue); + } } + error.raise(); } if (mgmtExchange != 0) @@ -115,7 +161,7 @@ void Exchange::routeIVE(){ Exchange::Exchange (const string& _name, Manageable* parent, Broker* b) : name(_name), durable(false), persistenceId(0), sequence(false), - sequenceNo(0), ive(false), mgmtExchange(0), broker(b) + sequenceNo(0), ive(false), mgmtExchange(0), broker(b), destroyed(false) { if (parent != 0 && broker != 0) { @@ -133,7 +179,7 @@ Exchange::Exchange (const string& _name, Manageable* parent, Broker* b) : Exchange::Exchange(const string& _name, bool _durable, const qpid::framing::FieldTable& _args, Manageable* parent, Broker* b) : name(_name), durable(_durable), alternateUsers(0), persistenceId(0), - args(_args), sequence(false), sequenceNo(0), ive(false), mgmtExchange(0), broker(b) + args(_args), sequence(false), sequenceNo(0), ive(false), mgmtExchange(0), broker(b), destroyed(false) { if (parent != 0 && broker != 0) { @@ -155,7 +201,11 @@ Exchange::Exchange(const string& _name, bool _durable, const qpid::framing::Fiel } ive = _args.get(qpidIVE); - if (ive) QPID_LOG(debug, "Configured exchange " << _name << " with Initial Value"); + if (ive) { + if (broker && broker->isInCluster()) + throw framing::NotImplementedException("Cannot use Initial Value Exchanges in a cluster"); + QPID_LOG(debug, "Configured exchange " << _name << " with Initial Value"); + } } Exchange::~Exchange () diff --git a/qpid/cpp/src/qpid/broker/Exchange.h b/qpid/cpp/src/qpid/broker/Exchange.h index 9c4e6be..e853155 100644 --- a/qpid/cpp/src/qpid/broker/Exchange.h +++ b/qpid/cpp/src/qpid/broker/Exchange.h @@ -10,9 +10,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -82,15 +82,15 @@ protected: private: Exchange* parent; }; - + typedef boost::shared_ptr<const std::vector<boost::shared_ptr<qpid::broker::Exchange::Binding> > > ConstBindingList; typedef boost::shared_ptr< std::vector<boost::shared_ptr<qpid::broker::Exchange::Binding> > > BindingList; void doRoute(Deliverable& msg, ConstBindingList b); void routeIVE(); - + struct MatchQueue { - const boost::shared_ptr<Queue> queue; + const boost::shared_ptr<Queue> queue; MatchQueue(boost::shared_ptr<Queue> q); bool operator()(Exchange::Binding::shared_ptr b); }; @@ -133,15 +133,15 @@ protected: /** Returns true if propagation is needed. */ bool delOrigin(const std::string& queueName, const std::string& origin){ - fedBindings[queueName].erase(origin); - return true; - } - - /** Returns true if propagation is needed. */ - bool delOrigin() { - if (localBindings > 0) - localBindings--; - return localBindings == 0; + if (origin.empty()) { // no remote == local binding + if (localBindings > 0) + localBindings--; + return localBindings == 0; + } + size_t match = fedBindings[queueName].erase(origin); + if (fedBindings[queueName].empty()) + fedBindings.erase(queueName); + return match != 0; } uint32_t count() { @@ -149,7 +149,11 @@ protected: } uint32_t countFedBindings(const std::string& queueName) { - return fedBindings[queueName].size(); + // don't use '[]' - it may increase size of fedBindings! + std::map<std::string, originSet>::iterator i; + if ((i = fedBindings.find(queueName)) != fedBindings.end()) + return i->second.size(); + return 0; } }; @@ -191,7 +195,7 @@ public: virtual bool isBound(boost::shared_ptr<Queue> queue, const std::string* const routingKey, const qpid::framing::FieldTable* const args) = 0; QPID_BROKER_EXTERN virtual void setProperties(const boost::intrusive_ptr<Message>&); virtual void route(Deliverable& msg, const std::string& routingKey, const qpid::framing::FieldTable* args) = 0; - + //PersistableExchange: QPID_BROKER_EXTERN void setPersistenceId(uint64_t id) const; uint64_t getPersistenceId() const { return persistenceId; } @@ -224,14 +228,18 @@ public: bool routeWithAlternate(Deliverable& message); + void destroy() { destroyed = true; } + bool isDestroyed() const { return destroyed; } + protected: qpid::sys::Mutex bridgeLock; std::vector<DynamicBridge*> bridgeVector; Broker* broker; + bool destroyed; QPID_BROKER_EXTERN virtual void handleHelloRequest(); void propagateFedOp(const std::string& routingKey, const std::string& tags, - const std::string& op, const std::string& origin, + const std::string& op, const std::string& origin, qpid::framing::FieldTable* extra_args=0); }; diff --git a/qpid/cpp/src/qpid/broker/ExchangeRegistry.cpp b/qpid/cpp/src/qpid/broker/ExchangeRegistry.cpp index 99b121c..1c8d26c 100644 --- a/qpid/cpp/src/qpid/broker/ExchangeRegistry.cpp +++ b/qpid/cpp/src/qpid/broker/ExchangeRegistry.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -39,7 +39,7 @@ pair<Exchange::shared_ptr, bool> ExchangeRegistry::declare(const string& name, c return declare(name, type, false, FieldTable()); } -pair<Exchange::shared_ptr, bool> ExchangeRegistry::declare(const string& name, const string& type, +pair<Exchange::shared_ptr, bool> ExchangeRegistry::declare(const string& name, const string& type, bool durable, const FieldTable& args){ RWlock::ScopedWlock locker(lock); ExchangeMap::iterator i = exchanges.find(name); @@ -61,7 +61,7 @@ pair<Exchange::shared_ptr, bool> ExchangeRegistry::declare(const string& name, c }else{ FunctionMap::iterator i = factory.find(type); if (i == factory.end()) { - throw UnknownExchangeTypeException(); + throw UnknownExchangeTypeException(); } else { exchange = i->second(name, durable, args, parent, broker); } @@ -82,6 +82,7 @@ void ExchangeRegistry::destroy(const string& name){ RWlock::ScopedWlock locker(lock); ExchangeMap::iterator i = exchanges.find(name); if (i != exchanges.end()) { + i->second->destroy(); exchanges.erase(i); } } @@ -104,7 +105,7 @@ void ExchangeRegistry::registerType(const std::string& type, FactoryFunction f) } -namespace +namespace { const std::string empty; } diff --git a/qpid/cpp/src/qpid/broker/FanOutExchange.cpp b/qpid/cpp/src/qpid/broker/FanOutExchange.cpp index ac2c914..5879fa0 100644 --- a/qpid/cpp/src/qpid/broker/FanOutExchange.cpp +++ b/qpid/cpp/src/qpid/broker/FanOutExchange.cpp @@ -18,6 +18,7 @@ * under the License. * */ +#include "qpid/log/Statement.h" #include "qpid/broker/FanOutExchange.h" #include "qpid/broker/FedOps.h" #include <algorithm> @@ -65,7 +66,7 @@ bool FanOutExchange::bind(Queue::shared_ptr queue, const string& /*key*/, const } else if (fedOp == fedOpUnbind) { propagate = fedBinding.delOrigin(queue->getName(), fedOrigin); if (fedBinding.countFedBindings(queue->getName()) == 0) - unbind(queue, "", 0); + unbind(queue, "", args); } else if (fedOp == fedOpReorigin) { if (fedBinding.hasLocal()) { propagateFedOp(string(), string(), fedOpBind, string()); @@ -78,12 +79,16 @@ bool FanOutExchange::bind(Queue::shared_ptr queue, const string& /*key*/, const return true; } -bool FanOutExchange::unbind(Queue::shared_ptr queue, const string& /*key*/, const FieldTable* /*args*/) +bool FanOutExchange::unbind(Queue::shared_ptr queue, const string& /*key*/, const FieldTable* args) { + string fedOrigin(args ? args->getAsString(qpidFedOrigin) : ""); bool propagate = false; + QPID_LOG(debug, "Unbinding queue " << queue->getName() + << " from exchange " << getName() << " origin=" << fedOrigin << ")" ); + if (bindings.remove_if(MatchQueue(queue))) { - propagate = fedBinding.delOrigin(); + propagate = fedBinding.delOrigin(queue->getName(), fedOrigin); if (mgmtExchange != 0) { mgmtExchange->dec_bindingCount(); } diff --git a/qpid/cpp/src/qpid/broker/HeadersExchange.cpp b/qpid/cpp/src/qpid/broker/HeadersExchange.cpp index 82ac591..abcaa5f 100644 --- a/qpid/cpp/src/qpid/broker/HeadersExchange.cpp +++ b/qpid/cpp/src/qpid/broker/HeadersExchange.cpp @@ -158,12 +158,13 @@ bool HeadersExchange::bind(Queue::shared_ptr queue, const string& bindingKey, co return true; } -bool HeadersExchange::unbind(Queue::shared_ptr queue, const string& bindingKey, const FieldTable*){ +bool HeadersExchange::unbind(Queue::shared_ptr queue, const string& bindingKey, const FieldTable *args){ bool propagate = false; + string fedOrigin(args ? args->getAsString(qpidFedOrigin) : ""); { Mutex::ScopedLock l(lock); - FedUnbindModifier modifier; + FedUnbindModifier modifier(queue->getName(), fedOrigin); MatchKey match_key(queue, bindingKey); bindings.modify_if(match_key, modifier); propagate = modifier.shouldPropagate; @@ -330,11 +331,7 @@ HeadersExchange::FedUnbindModifier::FedUnbindModifier() : shouldUnbind(false), s bool HeadersExchange::FedUnbindModifier::operator()(BoundKey & bk) { - if ("" == fedOrigin) { - shouldPropagate = bk.fedBinding.delOrigin(); - } else { - shouldPropagate = bk.fedBinding.delOrigin(queueName, fedOrigin); - } + shouldPropagate = bk.fedBinding.delOrigin(queueName, fedOrigin); if (bk.fedBinding.countFedBindings(queueName) == 0) { shouldUnbind = true; diff --git a/qpid/cpp/src/qpid/broker/Link.cpp b/qpid/cpp/src/qpid/broker/Link.cpp index f3acf7c..91861ad 100644 --- a/qpid/cpp/src/qpid/broker/Link.cpp +++ b/qpid/cpp/src/qpid/broker/Link.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -30,7 +30,6 @@ #include "qpid/framing/enum.h" #include "qpid/framing/reply_exceptions.h" #include "qpid/broker/AclModule.h" -#include "qpid/sys/ClusterSafe.h" using namespace qpid::broker; using qpid::framing::Buffer; @@ -57,8 +56,8 @@ Link::Link(LinkRegistry* _links, string& _password, Broker* _broker, Manageable* parent) - : links(_links), store(_store), host(_host), port(_port), - transport(_transport), + : links(_links), store(_store), host(_host), port(_port), + transport(_transport), durable(_durable), authMechanism(_authMechanism), username(_username), password(_password), persistenceId(0), mgmtObject(0), broker(_broker), state(0), @@ -97,7 +96,8 @@ void Link::setStateLH (int newState) return; state = newState; - if (mgmtObject == 0) + + if (hideManagement()) return; switch (state) @@ -122,7 +122,7 @@ void Link::startConnectionLH () QPID_LOG (debug, "Inter-broker link connecting to " << host << ":" << port); } catch(std::exception& e) { setStateLH(STATE_WAITING); - if (mgmtObject != 0) + if (!hideManagement()) mgmtObject->set_lastError (e.what()); } } @@ -133,8 +133,7 @@ void Link::established () addr << host << ":" << port; QPID_LOG (info, "Inter-broker link established to " << addr.str()); - // Don't raise the management event in a cluster, other members wont't get this call. - if (broker && !broker->isInCluster()) + if (!hideManagement() && agent) agent->raiseEvent(_qmf::EventBrokerLinkUp(addr.str())); { @@ -154,12 +153,11 @@ void Link::closed (int, std::string text) connection = 0; - // Don't raise the management event in a cluster, other members wont't get this call. if (state == STATE_OPERATIONAL) { stringstream addr; addr << host << ":" << port; QPID_LOG (warning, "Inter-broker link disconnected from " << addr.str()); - if (broker && !broker->isInCluster()) + if (!hideManagement() && agent) agent->raiseEvent(_qmf::EventBrokerLinkDown(addr.str())); } @@ -172,7 +170,7 @@ void Link::closed (int, std::string text) if (state != STATE_FAILED) { setStateLH(STATE_WAITING); - if (mgmtObject != 0) + if (!hideManagement()) mgmtObject->set_lastError (text); } @@ -221,7 +219,7 @@ void Link::cancel(Bridge::shared_ptr bridge) { { Mutex::ScopedLock mutex(lock); - + for (Bridges::iterator i = created.begin(); i != created.end(); i++) { if ((*i).get() == bridge.get()) { created.erase(i); @@ -277,9 +275,9 @@ void Link::maintenanceVisit () { Mutex::ScopedLock mutex(lock); - if (connection && updateUrls) { + if (connection && updateUrls) { urls.reset(connection->getKnownHosts()); - QPID_LOG(debug, "Known hosts for peer of inter-broker link: " << urls); + QPID_LOG(debug, "Known hosts for peer of inter-broker link: " << urls); updateUrls = false; } @@ -309,7 +307,7 @@ void Link::reconnect(const qpid::Address& a) port = a.port; transport = a.protocol; startConnectionLH(); - if (mgmtObject != 0) { + if (!hideManagement()) { stringstream errorString; errorString << "Failed over to " << a; mgmtObject->set_lastError(errorString.str()); @@ -319,7 +317,7 @@ void Link::reconnect(const qpid::Address& a) bool Link::tryFailover() { Address next; - if (urls.next(next) && + if (urls.next(next) && (next.host != host || next.port != port || next.protocol != transport)) { links->changeAddress(Address(transport, host, port), next); QPID_LOG(debug, "Link failing over to " << host << ":" << port); @@ -329,6 +327,12 @@ bool Link::tryFailover() } } +// Management updates for a linke are inconsistent in a cluster, so they are +// suppressed. +bool Link::hideManagement() const { + return !mgmtObject || ( broker && broker->isInCluster()); +} + uint Link::nextChannel() { Mutex::ScopedLock mutex(lock); @@ -341,7 +345,7 @@ void Link::notifyConnectionForced(const string text) Mutex::ScopedLock mutex(lock); setStateLH(STATE_FAILED); - if (mgmtObject != 0) + if (!hideManagement()) mgmtObject->set_lastError(text); } @@ -363,7 +367,7 @@ Link::shared_ptr Link::decode(LinkRegistry& links, Buffer& buffer) string authMechanism; string username; string password; - + buffer.getShortString(host); port = buffer.getShort(); buffer.getShortString(transport); @@ -375,7 +379,7 @@ Link::shared_ptr Link::decode(LinkRegistry& links, Buffer& buffer) return links.declare(host, port, transport, durable, authMechanism, username, password).first; } -void Link::encode(Buffer& buffer) const +void Link::encode(Buffer& buffer) const { buffer.putShortString(string("link")); buffer.putShortString(host); @@ -387,8 +391,8 @@ void Link::encode(Buffer& buffer) const buffer.putShortString(password); } -uint32_t Link::encodedSize() const -{ +uint32_t Link::encodedSize() const +{ return host.size() + 1 // short-string (host) + 5 // short-string ("link") + 2 // port diff --git a/qpid/cpp/src/qpid/broker/Link.h b/qpid/cpp/src/qpid/broker/Link.h index 75a680f..4badd8b 100644 --- a/qpid/cpp/src/qpid/broker/Link.h +++ b/qpid/cpp/src/qpid/broker/Link.h @@ -10,9 +10,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -85,6 +85,7 @@ namespace qpid { void destroy(); // Called when mgmt deletes this link void ioThreadProcessing(); // Called on connection's IO thread by request bool tryFailover(); // Called during maintenance visit + bool hideManagement() const; public: typedef boost::shared_ptr<Link> shared_ptr; @@ -122,12 +123,12 @@ namespace qpid { void notifyConnectionForced(const std::string text); void setPassive(bool p); - + // PersistableConfig: void setPersistenceId(uint64_t id) const; uint64_t getPersistenceId() const { return persistenceId; } uint32_t encodedSize() const; - void encode(framing::Buffer& buffer) const; + void encode(framing::Buffer& buffer) const; const std::string& getName() const; static Link::shared_ptr decode(LinkRegistry& links, framing::Buffer& buffer); @@ -135,6 +136,7 @@ namespace qpid { // Manageable entry points management::ManagementObject* GetManagementObject(void) const; management::Manageable::status_t ManagementMethod(uint32_t, management::Args&, std::string&); + }; } } diff --git a/qpid/cpp/src/qpid/broker/LinkRegistry.cpp b/qpid/cpp/src/qpid/broker/LinkRegistry.cpp index 7b1c75d..e9885f5 100644 --- a/qpid/cpp/src/qpid/broker/LinkRegistry.cpp +++ b/qpid/cpp/src/qpid/broker/LinkRegistry.cpp @@ -7,9 +7,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -381,7 +381,7 @@ std::string LinkRegistry::createKey(const std::string& host, uint16_t port) { return keystream.str(); } -void LinkRegistry::setPassive(bool p) +void LinkRegistry::setPassive(bool p) { Mutex::ScopedLock locker(lock); passiveChanged = p != passive; diff --git a/qpid/cpp/src/qpid/broker/NullMessageStore.cpp b/qpid/cpp/src/qpid/broker/NullMessageStore.cpp index dc8615d..43f600e 100644 --- a/qpid/cpp/src/qpid/broker/NullMessageStore.cpp +++ b/qpid/cpp/src/qpid/broker/NullMessageStore.cpp @@ -126,21 +126,25 @@ std::auto_ptr<TPCTransactionContext> NullMessageStore::begin(const std::string& void NullMessageStore::prepare(TPCTransactionContext& ctxt) { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(lock); prepared.insert(DummyCtxt::getXid(ctxt)); } void NullMessageStore::commit(TransactionContext& ctxt) { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(lock); prepared.erase(DummyCtxt::getXid(ctxt)); } void NullMessageStore::abort(TransactionContext& ctxt) { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(lock); prepared.erase(DummyCtxt::getXid(ctxt)); } void NullMessageStore::collectPreparedXids(std::set<std::string>& out) { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(lock); out.insert(prepared.begin(), prepared.end()); } diff --git a/qpid/cpp/src/qpid/broker/NullMessageStore.h b/qpid/cpp/src/qpid/broker/NullMessageStore.h index e148ec4..5f51d53 100644 --- a/qpid/cpp/src/qpid/broker/NullMessageStore.h +++ b/qpid/cpp/src/qpid/broker/NullMessageStore.h @@ -25,6 +25,7 @@ #include "qpid/broker/BrokerImportExport.h" #include "qpid/broker/MessageStore.h" #include "qpid/broker/Queue.h" +#include "qpid/sys/Mutex.h" #include <boost/intrusive_ptr.hpp> @@ -38,6 +39,7 @@ class NullMessageStore : public MessageStore { std::set<std::string> prepared; uint64_t nextPersistenceId; + qpid::sys::Mutex lock; public: QPID_BROKER_EXTERN NullMessageStore(); diff --git a/qpid/cpp/src/qpid/broker/Queue.cpp b/qpid/cpp/src/qpid/broker/Queue.cpp index 7a266fb..8efa8be 100644 --- a/qpid/cpp/src/qpid/broker/Queue.cpp +++ b/qpid/cpp/src/qpid/broker/Queue.cpp @@ -183,7 +183,6 @@ void Queue::recover(boost::intrusive_ptr<Message>& msg){ // setup synclist for recovered messages, so they don't get re-stored on lastNodeFailure msg->addToSyncList(shared_from_this(), store); } - msg->enqueueComplete(); // mark the message as enqueued if (store && (!msg->isContentLoaded() || msg->checkContentReleasable())) { //content has not been loaded, need to ensure that lazy loading mode is set: @@ -210,7 +209,6 @@ void Queue::requeue(const QueuedMessage& msg){ { Mutex::ScopedLock locker(messageLock); if (!isEnqueued(msg)) return; - msg.payload->enqueueComplete(); // mark the message as enqueued messages->reinsert(msg); listeners.populate(copy); @@ -632,7 +630,9 @@ bool Queue::enqueue(TransactionContext* ctxt, boost::intrusive_ptr<Message>& msg } if ((msg->isPersistent() || msg->checkContentReleasable()) && store) { - msg->enqueueAsync(shared_from_this(), store); //increment to async counter -- for message sent to more than one queue + // mark the message as being enqueued - the store MUST CALL msg->enqueueComplete() + // when it considers the message stored. + msg->enqueueAsync(shared_from_this(), store); boost::intrusive_ptr<PersistableMessage> pmsg = boost::static_pointer_cast<PersistableMessage>(msg); store->enqueue(ctxt, pmsg, *this); return true; diff --git a/qpid/cpp/src/qpid/broker/Queue.h b/qpid/cpp/src/qpid/broker/Queue.h index 73d52ec..c4f1bcc 100644 --- a/qpid/cpp/src/qpid/broker/Queue.h +++ b/qpid/cpp/src/qpid/broker/Queue.h @@ -348,6 +348,11 @@ class Queue : public boost::enable_shared_from_this<Queue>, bindings.eachBinding(f); } + /** Apply f to each Observer on the queue */ + template <class F> void eachObserver(F f) { + std::for_each<Observers::iterator, F>(observers.begin(), observers.end(), f); + } + /** Set the position sequence number for the next message on the queue. * Must be >= the current sequence number. * Used by cluster to replicate queues. diff --git a/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp b/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp index 3494288..b2e2e54 100644 --- a/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp +++ b/qpid/cpp/src/qpid/broker/QueueFlowLimit.cpp @@ -92,7 +92,7 @@ namespace { QueueFlowLimit::QueueFlowLimit(Queue *_queue, uint32_t _flowStopCount, uint32_t _flowResumeCount, uint64_t _flowStopSize, uint64_t _flowResumeSize) - : queue(_queue), queueName("<unknown>"), + : StatefulQueueObserver(std::string("QueueFlowLimit")), queue(_queue), queueName("<unknown>"), flowStopCount(_flowStopCount), flowResumeCount(_flowResumeCount), flowStopSize(_flowStopSize), flowResumeSize(_flowResumeSize), flowStopped(false), count(0), size(0), queueMgmtObj(0), broker(0) @@ -120,11 +120,24 @@ QueueFlowLimit::QueueFlowLimit(Queue *_queue, } +QueueFlowLimit::~QueueFlowLimit() +{ + sys::Mutex::ScopedLock l(indexLock); + if (!index.empty()) { + // we're gone - release all pending msgs + for (std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::iterator itr = index.begin(); + itr != index.end(); ++itr) + if (itr->second) + try { + itr->second->getIngressCompletion().finishCompleter(); + } catch (...) {} // ignore - not safe for a destructor to throw. + index.clear(); + } +} + void QueueFlowLimit::enqueued(const QueuedMessage& msg) { - if (!msg.payload) return; - sys::Mutex::ScopedLock l(indexLock); ++count; @@ -152,7 +165,9 @@ void QueueFlowLimit::enqueued(const QueuedMessage& msg) } QPID_LOG(trace, "Queue \"" << queueName << "\": setting flow control for msg pos=" << msg.position); msg.payload->getIngressCompletion().startCompleter(); // don't complete until flow resumes - index.insert(msg.payload); + bool unique; + unique = index.insert(std::pair<framing::SequenceNumber, boost::intrusive_ptr<Message> >(msg.position, msg.payload)).second; + assert(unique); } } @@ -160,8 +175,6 @@ void QueueFlowLimit::enqueued(const QueuedMessage& msg) void QueueFlowLimit::dequeued(const QueuedMessage& msg) { - if (!msg.payload) return; - sys::Mutex::ScopedLock l(indexLock); if (count > 0) { @@ -189,16 +202,16 @@ void QueueFlowLimit::dequeued(const QueuedMessage& msg) if (!index.empty()) { if (!flowStopped) { // flow enabled - release all pending msgs - while (!index.empty()) { - std::set< boost::intrusive_ptr<Message> >::iterator itr = index.begin(); - (*itr)->getIngressCompletion().finishCompleter(); - index.erase(itr); - } + for (std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::iterator itr = index.begin(); + itr != index.end(); ++itr) + if (itr->second) + itr->second->getIngressCompletion().finishCompleter(); + index.clear(); } else { // even if flow controlled, we must release this msg as it is being dequeued - std::set< boost::intrusive_ptr<Message> >::iterator itr = index.find(msg.payload); + std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::iterator itr = index.find(msg.position); if (itr != index.end()) { // this msg is flow controlled, release it: - (*itr)->getIngressCompletion().finishCompleter(); + msg.payload->getIngressCompletion().finishCompleter(); index.erase(itr); } } @@ -206,34 +219,6 @@ void QueueFlowLimit::dequeued(const QueuedMessage& msg) } -/** used by clustering: is the given message's completion blocked due to flow - * control? True if message is blocked. (for the clustering updater: done - * after msgs have been replicated to the updatee). - */ -bool QueueFlowLimit::getState(const QueuedMessage& msg) const -{ - sys::Mutex::ScopedLock l(indexLock); - return (index.find(msg.payload) != index.end()); -} - - -/** artificially force the flow control state of a given message - * (for the clustering updatee: done after msgs have been replicated to - * the updatee's queue) - */ -void QueueFlowLimit::setState(const QueuedMessage& msg, bool blocked) -{ - if (blocked && msg.payload) { - - sys::Mutex::ScopedLock l(indexLock); - assert(index.find(msg.payload) == index.end()); - - QPID_LOG(debug, "Queue \"" << queue->getName() << "\": forcing flow control for msg pos=" << msg.position << " for CLUSTER SYNC"); - index.insert(msg.payload); - } -} - - void QueueFlowLimit::encode(Buffer& buffer) const { buffer.putLong(flowStopCount); @@ -281,7 +266,7 @@ void QueueFlowLimit::setDefaults(uint64_t maxQueueSize, uint flowStopRatio, uint defaultFlowStopRatio = flowStopRatio; defaultFlowResumeRatio = flowResumeRatio; - /** @todo Verify valid range on Broker::Options instead of here */ + /** @todo KAG: Verify valid range on Broker::Options instead of here */ if (flowStopRatio > 100 || flowResumeRatio > 100) throw InvalidArgumentException(QPID_MSG("Default queue flow ratios must be between 0 and 100, inclusive:" << " flowStopRatio=" << flowStopRatio @@ -312,7 +297,9 @@ QueueFlowLimit *QueueFlowLimit::createLimit(Queue *queue, const qpid::framing::F return 0; } - if (settings.get(flowStopCountKey) || settings.get(flowStopSizeKey)) { + if (settings.get(flowStopCountKey) || settings.get(flowStopSizeKey) || + settings.get(flowResumeCountKey) || settings.get(flowResumeSizeKey)) { + // user provided (some) flow settings manually... uint32_t flowStopCount = getCapacity(settings, flowStopCountKey, 0); uint32_t flowResumeCount = getCapacity(settings, flowResumeCountKey, 0); uint64_t flowStopSize = getCapacity(settings, flowStopSizeKey, 0); @@ -320,32 +307,88 @@ QueueFlowLimit *QueueFlowLimit::createLimit(Queue *queue, const qpid::framing::F if (flowStopCount == 0 && flowStopSize == 0) { // disable flow control return 0; } - /** @todo KAG - remove once cluster support for flow control done. */ - // TODO aconway 2011-02-16: is queue==0 only in tests? - // TODO kgiusti 2011-02-19: yes! The unit tests test this class in isolation */ - if (queue && queue->getBroker() && queue->getBroker()->isInCluster()) { - QPID_LOG(warning, "Producer Flow Control TBD for clustered brokers - queue flow control disabled for queue " - << queue->getName()); - return 0; - } return new QueueFlowLimit(queue, flowStopCount, flowResumeCount, flowStopSize, flowResumeSize); } - if (defaultFlowStopRatio) { + if (defaultFlowStopRatio) { // broker has a default ratio setup... uint64_t maxByteCount = getCapacity(settings, QueuePolicy::maxSizeKey, defaultMaxSize); uint64_t flowStopSize = (uint64_t)(maxByteCount * (defaultFlowStopRatio/100.0) + 0.5); uint64_t flowResumeSize = (uint64_t)(maxByteCount * (defaultFlowResumeRatio/100.0)); + uint32_t maxMsgCount = getCapacity(settings, QueuePolicy::maxCountKey, 0); // no size by default + uint32_t flowStopCount = (uint32_t)(maxMsgCount * (defaultFlowStopRatio/100.0) + 0.5); + uint32_t flowResumeCount = (uint32_t)(maxMsgCount * (defaultFlowResumeRatio/100.0)); - /** todo KAG - remove once cluster support for flow control done. */ - if (queue && queue->getBroker() && queue->getBroker()->isInCluster()) { - QPID_LOG(warning, "Producer Flow Control TBD for clustered brokers - queue flow control disabled for queue " - << queue->getName()); - return 0; + return new QueueFlowLimit(queue, flowStopCount, flowResumeCount, flowStopSize, flowResumeSize); + } + return 0; +} + +/* Cluster replication */ + +namespace { + /** pack a set of sequence number ranges into a framing::Array */ + void buildSeqRangeArray(qpid::framing::Array *seqs, + const qpid::framing::SequenceNumber& first, + const qpid::framing::SequenceNumber& last) + { + seqs->push_back(qpid::framing::Array::ValuePtr(new Unsigned32Value(first))); + seqs->push_back(qpid::framing::Array::ValuePtr(new Unsigned32Value(last))); + } +} + +/** Runs on UPDATER to snapshot current state */ +void QueueFlowLimit::getState(qpid::framing::FieldTable& state ) const +{ + sys::Mutex::ScopedLock l(indexLock); + state.clear(); + + framing::SequenceSet ss; + if (!index.empty()) { + /* replicate the set of messages pending flow control */ + for (std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> >::const_iterator itr = index.begin(); + itr != index.end(); ++itr) { + ss.add(itr->first); } + framing::Array seqs(TYPE_CODE_UINT32); + typedef boost::function<void(framing::SequenceNumber, framing::SequenceNumber)> arrayBuilder; + ss.for_each((arrayBuilder)boost::bind(&buildSeqRangeArray, &seqs, _1, _2)); + state.setArray("pendingMsgSeqs", seqs); + } + QPID_LOG(debug, "Queue \"" << queueName << "\": flow limit replicating pending msgs, range=" << ss); +} + - return new QueueFlowLimit(queue, 0, 0, flowStopSize, flowResumeSize); +/** called on UPDATEE to set state from snapshot */ +void QueueFlowLimit::setState(const qpid::framing::FieldTable& state) +{ + sys::Mutex::ScopedLock l(indexLock); + index.clear(); + + framing::SequenceSet fcmsg; + framing::Array seqArray(TYPE_CODE_UINT32); + if (state.getArray("pendingMsgSeqs", seqArray)) { + assert((seqArray.count() & 0x01) == 0); // must be even since they are sequence ranges + framing::Array::const_iterator i = seqArray.begin(); + while (i != seqArray.end()) { + framing::SequenceNumber first((*i)->getIntegerValue<uint32_t, 4>()); + ++i; + framing::SequenceNumber last((*i)->getIntegerValue<uint32_t, 4>()); + ++i; + fcmsg.add(first, last); + for (SequenceNumber seq = first; seq <= last; ++seq) { + QueuedMessage msg(queue->find(seq)); // fyi: msg.payload may be null if msg is delivered & unacked + bool unique; + unique = index.insert(std::pair<framing::SequenceNumber, boost::intrusive_ptr<Message> >(seq, msg.payload)).second; + assert(unique); + } + } } - return 0; + + flowStopped = index.size() != 0; + if (queueMgmtObj) { + queueMgmtObj->set_flowStopped(isFlowControlActive()); + } + QPID_LOG(debug, "Queue \"" << queueName << "\": flow limit replicated the pending msgs, range=" << fcmsg) } diff --git a/qpid/cpp/src/qpid/broker/QueueFlowLimit.h b/qpid/cpp/src/qpid/broker/QueueFlowLimit.h index 69d91df..c02e479 100644 --- a/qpid/cpp/src/qpid/broker/QueueFlowLimit.h +++ b/qpid/cpp/src/qpid/broker/QueueFlowLimit.h @@ -27,7 +27,7 @@ #include <memory> #include "qpid/broker/BrokerImportExport.h" #include "qpid/broker/QueuedMessage.h" -#include "qpid/broker/QueueObserver.h" +#include "qpid/broker/StatefulQueueObserver.h" #include "qpid/framing/FieldTable.h" #include "qpid/sys/AtomicValue.h" #include "qpid/sys/Mutex.h" @@ -53,7 +53,7 @@ class Broker; * passing _either_ level may turn flow control ON, but _both_ must be * below level before flow control will be turned OFF. */ - class QueueFlowLimit : public QueueObserver + class QueueFlowLimit : public StatefulQueueObserver { static uint64_t defaultMaxSize; static uint defaultFlowStopRatio; @@ -78,7 +78,7 @@ class Broker; static QPID_BROKER_EXTERN const std::string flowStopSizeKey; static QPID_BROKER_EXTERN const std::string flowResumeSizeKey; - virtual ~QueueFlowLimit() {} + QPID_BROKER_EXTERN virtual ~QueueFlowLimit(); /** the queue has added QueuedMessage. Returns true if flow state changes */ QPID_BROKER_EXTERN void enqueued(const QueuedMessage&); @@ -86,9 +86,8 @@ class Broker; QPID_BROKER_EXTERN void dequeued(const QueuedMessage&); /** for clustering: */ - /** true if the given message is flow controlled, and cannot be completed. */ - bool getState(const QueuedMessage&) const; - void setState(const QueuedMessage&, bool blocked); + QPID_BROKER_EXTERN void getState(qpid::framing::FieldTable&) const; + QPID_BROKER_EXTERN void setState(const qpid::framing::FieldTable&); uint32_t getFlowStopCount() const { return flowStopCount; } uint32_t getFlowResumeCount() const { return flowResumeCount; } @@ -111,7 +110,7 @@ class Broker; protected: // msgs waiting for flow to become available. - std::set< boost::intrusive_ptr<Message> > index; + std::map<framing::SequenceNumber, boost::intrusive_ptr<Message> > index; mutable qpid::sys::Mutex indexLock; _qmfBroker::Queue *queueMgmtObj; diff --git a/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp b/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp index 38cb804..cd67353 100644 --- a/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp +++ b/qpid/cpp/src/qpid/broker/RecoveredDequeue.cpp @@ -43,7 +43,6 @@ void RecoveredDequeue::commit() throw() void RecoveredDequeue::rollback() throw() { - msg->enqueueComplete(); queue->process(msg); } diff --git a/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp b/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp index 6263c63..6d2eaee 100644 --- a/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp +++ b/qpid/cpp/src/qpid/broker/RecoveredEnqueue.cpp @@ -36,7 +36,6 @@ bool RecoveredEnqueue::prepare(TransactionContext*) throw(){ } void RecoveredEnqueue::commit() throw(){ - msg->enqueueComplete(); queue->process(msg); } diff --git a/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp b/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp index 6f8b125..d084096 100644 --- a/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp +++ b/qpid/cpp/src/qpid/broker/RecoveryManagerImpl.cpp @@ -252,7 +252,6 @@ void RecoverableMessageImpl::dequeue(DtxBuffer::shared_ptr buffer, Queue::shared void RecoverableMessageImpl::enqueue(DtxBuffer::shared_ptr buffer, Queue::shared_ptr queue) { - msg->enqueueComplete(); // recoved nmessage to enqueued in store already buffer->enlist(TxOp::shared_ptr(new RecoveredEnqueue(queue, msg))); } diff --git a/qpid/cpp/src/qpid/broker/SemanticState.cpp b/qpid/cpp/src/qpid/broker/SemanticState.cpp index ba1f989..ce86253 100644 --- a/qpid/cpp/src/qpid/broker/SemanticState.cpp +++ b/qpid/cpp/src/qpid/broker/SemanticState.cpp @@ -462,7 +462,7 @@ void SemanticState::route(intrusive_ptr<Message> msg, Deliverable& strategy) { msg->setTimestamp(getSession().getBroker().getExpiryPolicy()); std::string exchangeName = msg->getExchangeName(); - if (!cacheExchange || cacheExchange->getName() != exchangeName) + if (!cacheExchange || cacheExchange->getName() != exchangeName || cacheExchange->isDestroyed()) cacheExchange = session.getBroker().getExchanges().get(exchangeName); cacheExchange->setProperties(msg); diff --git a/qpid/cpp/src/qpid/broker/SessionState.cpp b/qpid/cpp/src/qpid/broker/SessionState.cpp index 18dbf63..957d5bd 100644 --- a/qpid/cpp/src/qpid/broker/SessionState.cpp +++ b/qpid/cpp/src/qpid/broker/SessionState.cpp @@ -95,14 +95,13 @@ void SessionState::addManagementObject() { } SessionState::~SessionState() { + asyncCommandCompleter->cancel(); semanticState.closed(); if (mgmtObject != 0) mgmtObject->resourceDestroy (); if (flowControlTimer) flowControlTimer->cancel(); - - asyncCommandCompleter->cancel(); } AMQP_ClientProxy& SessionState::getProxy() { @@ -127,6 +126,7 @@ bool SessionState::isLocal(const ConnectionToken* t) const void SessionState::detach() { QPID_LOG(debug, getId() << ": detached on broker."); + asyncCommandCompleter->detached(); disableOutput(); handler = 0; if (mgmtObject != 0) @@ -147,6 +147,7 @@ void SessionState::attach(SessionHandler& h) { mgmtObject->set_connectionRef (h.getConnection().GetManagementObject()->getObjectId()); mgmtObject->set_channelId (h.getChannel()); } + asyncCommandCompleter->attached(); } void SessionState::abort() { @@ -426,6 +427,7 @@ void SessionState::addPendingExecutionSync() if (receiverGetIncomplete().front() < syncCommandId) { currentCommandComplete = false; pendingExecutionSyncs.push(syncCommandId); + asyncCommandCompleter->flushPendingMessages(); QPID_LOG(debug, getId() << ": delaying completion of execution.sync " << syncCommandId); } } @@ -438,6 +440,17 @@ boost::intrusive_ptr<AsyncCompletion::Callback> SessionState::IncompleteIngressMsgXfer::clone() { boost::intrusive_ptr<SessionState::IncompleteIngressMsgXfer> cb(new SessionState::IncompleteIngressMsgXfer(session, msg)); + + // Optimization: this routine is *only* invoked when the message needs to be asynchronously completed. + // If the client is pending the message.transfer completion, flush now to force immediate write to journal. + if (requiresSync) + msg->flush(); + else { + // otherwise, we need to track this message in order to flush it if an execution.sync arrives + // before it has been completed (see flushPendingMessages()) + pending = true; + completerContext->addPendingMessage(msg); + } return cb; } @@ -448,17 +461,18 @@ SessionState::IncompleteIngressMsgXfer::clone() */ void SessionState::IncompleteIngressMsgXfer::completed(bool sync) { + if (pending) completerContext->deletePendingMessage(id); if (!sync) { /** note well: this path may execute in any thread. It is safe to access * the scheduledCompleterContext, since *this has a shared pointer to it. - * but not session or msg! + * but not session! */ - session = 0; msg = 0; + session = 0; QPID_LOG(debug, ": async completion callback scheduled for msg seq=" << id); completerContext->scheduleMsgCompletion(id, requiresAccept, requiresSync); } else { // this path runs directly from the ac->end() call in handleContent() above, - // so *session and *msg are definately valid. + // so *session is definately valid. if (session->isAttached()) { QPID_LOG(debug, ": receive completed for msg seq=" << id); session->completeRcvMsg(id, requiresAccept, requiresSync); @@ -477,6 +491,40 @@ void SessionState::AsyncCommandCompleter::schedule(boost::intrusive_ptr<AsyncCom } +/** Track an ingress message that is pending completion */ +void SessionState::AsyncCommandCompleter::addPendingMessage(boost::intrusive_ptr<Message> msg) +{ + qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); + std::pair<SequenceNumber, boost::intrusive_ptr<Message> > item(msg->getCommandId(), msg); + bool unique = pendingMsgs.insert(item).second; + assert(unique); +} + + +/** pending message has completed */ +void SessionState::AsyncCommandCompleter::deletePendingMessage(SequenceNumber id) +{ + qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); + pendingMsgs.erase(id); +} + + +/** done when an execution.sync arrives */ +void SessionState::AsyncCommandCompleter::flushPendingMessages() +{ + std::map<SequenceNumber, boost::intrusive_ptr<Message> > copy; + { + qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); + pendingMsgs.swap(copy); // we've only tracked these in case a flush is needed, so nuke 'em now. + } + // drop lock, so it is safe to call "flush()" + for (std::map<SequenceNumber, boost::intrusive_ptr<Message> >::iterator i = copy.begin(); + i != copy.end(); ++i) { + i->second->flush(); + } +} + + /** mark an ingress Message.Transfer command as completed. * This method must be thread safe - it may run on any thread. */ @@ -486,7 +534,7 @@ void SessionState::AsyncCommandCompleter::scheduleMsgCompletion(SequenceNumber c { qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); - if (session) { + if (session && isAttached) { MessageInfo msg(cmd, requiresAccept, requiresSync); completedMsgs.push_back(msg); if (completedMsgs.size() == 1) { @@ -522,4 +570,22 @@ void SessionState::AsyncCommandCompleter::cancel() session = 0; } + +/** inform the completer that the session has attached, + * allows command completion scheduling from any thread */ +void SessionState::AsyncCommandCompleter::attached() +{ + qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); + isAttached = true; +} + + +/** inform the completer that the session has detached, + * disables command completion scheduling from any thread */ +void SessionState::AsyncCommandCompleter::detached() +{ + qpid::sys::ScopedLock<qpid::sys::Mutex> l(completerLock); + isAttached = false; +} + }} // namespace qpid::broker diff --git a/qpid/cpp/src/qpid/broker/SessionState.h b/qpid/cpp/src/qpid/broker/SessionState.h index 2250940..b43df0c 100644 --- a/qpid/cpp/src/qpid/broker/SessionState.h +++ b/qpid/cpp/src/qpid/broker/SessionState.h @@ -187,6 +187,7 @@ class SessionState : public qpid::SessionState, class AsyncCommandCompleter : public RefCounted { private: SessionState *session; + bool isAttached; qpid::sys::Mutex completerLock; // special-case message.transfer commands for optimization @@ -198,6 +199,10 @@ class SessionState : public qpid::SessionState, : cmd(c), requiresAccept(a), requiresSync(s) {} }; std::vector<MessageInfo> completedMsgs; + // If an ingress message does not require a Sync, we need to + // hold a reference to it in case an Execution.Sync command is received and we + // have to manually flush the message. + std::map<SequenceNumber, boost::intrusive_ptr<Message> > pendingMsgs; /** complete all pending commands, runs in IO thread */ void completeCommands(); @@ -205,15 +210,21 @@ class SessionState : public qpid::SessionState, /** for scheduling a run of "completeCommands()" on the IO thread */ static void schedule(boost::intrusive_ptr<AsyncCommandCompleter>); - public: - AsyncCommandCompleter(SessionState *s) : session(s) {}; + public: + AsyncCommandCompleter(SessionState *s) : session(s), isAttached(s->isAttached()) {}; ~AsyncCommandCompleter() {}; - /** schedule the completion of an ingress message.transfer command */ + /** track a message pending ingress completion */ + void addPendingMessage(boost::intrusive_ptr<Message> m); + void deletePendingMessage(SequenceNumber id); + void flushPendingMessages(); + /** schedule the processing of a completed ingress message.transfer command */ void scheduleMsgCompletion(SequenceNumber cmd, bool requiresAccept, bool requiresSync); void cancel(); // called by SessionState destructor. + void attached(); // called by SessionState on attach() + void detached(); // called by SessionState on detach() }; boost::intrusive_ptr<AsyncCommandCompleter> asyncCommandCompleter; @@ -240,20 +251,22 @@ class SessionState : public qpid::SessionState, IncompleteIngressMsgXfer( SessionState *ss, boost::intrusive_ptr<Message> m ) : AsyncCommandContext(ss, m->getCommandId()), - session(ss), - msg(m.get()), - requiresAccept(msg->requiresAccept()), - requiresSync(msg->getFrames().getMethod()->isSync()) {}; + session(ss), + msg(m), + requiresAccept(m->requiresAccept()), + requiresSync(m->getFrames().getMethod()->isSync()), + pending(false) {} virtual ~IncompleteIngressMsgXfer() {}; virtual void completed(bool); virtual boost::intrusive_ptr<AsyncCompletion::Callback> clone(); private: - SessionState *session; // only valid if sync == true - Message *msg; // only valid if sync == true + SessionState *session; // only valid if sync flag in callback is true + boost::intrusive_ptr<Message> msg; bool requiresAccept; bool requiresSync; + bool pending; // true if msg saved on pending list... }; friend class SessionManager; diff --git a/qpid/cpp/src/qpid/broker/StatefulQueueObserver.h b/qpid/cpp/src/qpid/broker/StatefulQueueObserver.h new file mode 100644 index 0000000..c682d46 --- /dev/null +++ b/qpid/cpp/src/qpid/broker/StatefulQueueObserver.h @@ -0,0 +1,63 @@ +#ifndef QPID_BROKER_STATEFULQUEUEOBSERVER_H +#define QPID_BROKER_STATEFULQUEUEOBSERVER_H + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#include "qpid/broker/QueueObserver.h" +#include "qpid/framing/FieldTable.h" + +namespace qpid { +namespace broker { + +/** + * Specialized type of QueueObserver that maintains internal state that has to + * be replicated across clustered brokers. + */ +class StatefulQueueObserver : public QueueObserver +{ + public: + StatefulQueueObserver(std::string _id) : id(_id) {} + virtual ~StatefulQueueObserver() {} + + /** This identifier must uniquely identify this particular observer amoung + * all observers on a queue. For cluster replication, this id will be used + * to identify the peer queue observer for synchronization across + * brokers. + */ + const std::string& getId() const { return id; } + + /** This method should return the observer's internal state as an opaque + * map. + */ + virtual void getState(qpid::framing::FieldTable& state ) const = 0; + + /** The input map represents the internal state of the peer observer that + * this observer should synchonize to. + */ + virtual void setState(const qpid::framing::FieldTable&) = 0; + + + private: + std::string id; +}; +}} // namespace qpid::broker + +#endif /*!QPID_BROKER_STATEFULQUEUEOBSERVER_H*/ diff --git a/qpid/cpp/src/qpid/broker/TopicExchange.cpp b/qpid/cpp/src/qpid/broker/TopicExchange.cpp index 1b0fe71..d8eb11e 100644 --- a/qpid/cpp/src/qpid/broker/TopicExchange.cpp +++ b/qpid/cpp/src/qpid/broker/TopicExchange.cpp @@ -249,21 +249,21 @@ bool TopicExchange::bind(Queue::shared_ptr queue, const string& routingKey, cons if (mgmtExchange != 0) { mgmtExchange->inc_bindingCount(); } - QPID_LOG(debug, "Bound key [" << routingPattern << "] to queue " << queue->getName() - << " (origin=" << fedOrigin << ")"); + QPID_LOG(debug, "Binding key [" << routingPattern << "] to queue " << queue->getName() + << " on exchange " << getName() << " (origin=" << fedOrigin << ")"); } } else if (fedOp == fedOpUnbind) { - bool reallyUnbind = false; - { - RWlock::ScopedWlock l(lock); - BindingKey* bk = bindingTree.getBindingKey(routingPattern); - if (bk) { - propagate = bk->fedBinding.delOrigin(queue->getName(), fedOrigin); - reallyUnbind = bk->fedBinding.countFedBindings(queue->getName()) == 0; + RWlock::ScopedWlock l(lock); + BindingKey* bk = getQueueBinding(queue, routingPattern); + if (bk) { + QPID_LOG(debug, "FedOpUnbind [" << routingPattern << "] from exchange " << getName() + << " on queue=" << queue->getName() << " origin=" << fedOrigin); + propagate = bk->fedBinding.delOrigin(queue->getName(), fedOrigin); + // if this was the last binding for the queue, delete the binding + if (bk->fedBinding.countFedBindings(queue->getName()) == 0) { + deleteBinding(queue, routingPattern, bk); } } - if (reallyUnbind) - unbind(queue, routingPattern, 0); } else if (fedOp == fedOpReorigin) { /** gather up all the keys that need rebinding in a local vector * while holding the lock. Then propagate once the lock is @@ -287,14 +287,30 @@ bool TopicExchange::bind(Queue::shared_ptr queue, const string& routingKey, cons return true; } -bool TopicExchange::unbind(Queue::shared_ptr queue, const string& constRoutingKey, const FieldTable* /*args*/){ +bool TopicExchange::unbind(Queue::shared_ptr queue, const string& constRoutingKey, const FieldTable* args) +{ + string fedOrigin(args ? args->getAsString(qpidFedOrigin) : ""); + QPID_LOG(debug, "Unbinding key [" << constRoutingKey << "] from queue " << queue->getName() + << " on exchange " << getName() << " origin=" << fedOrigin << ")" ); + RWlock::ScopedWlock l(lock); string routingKey = normalize(constRoutingKey); - BindingKey* bk = bindingTree.getBindingKey(routingKey); + BindingKey* bk = getQueueBinding(queue, routingKey); if (!bk) return false; - Binding::vector& qv(bk->bindingVector); - bool propagate = false; + bool propagate = bk->fedBinding.delOrigin(queue->getName(), fedOrigin); + deleteBinding(queue, routingKey, bk); + if (propagate) + propagateFedOp(routingKey, string(), fedOpUnbind, string()); + return true; +} + +bool TopicExchange::deleteBinding(Queue::shared_ptr queue, + const std::string& routingKey, + BindingKey *bk) +{ + // Note well: write lock held by caller + Binding::vector& qv(bk->bindingVector); Binding::vector::iterator q; for (q = qv.begin(); q != qv.end(); q++) if ((*q)->queue == queue) @@ -303,31 +319,32 @@ bool TopicExchange::unbind(Queue::shared_ptr queue, const string& constRoutingKe qv.erase(q); assert(nBindings > 0); nBindings--; - propagate = bk->fedBinding.delOrigin(); + if(qv.empty()) { bindingTree.removeBindingKey(routingKey); } if (mgmtExchange != 0) { mgmtExchange->dec_bindingCount(); } - QPID_LOG(debug, "Unbound [" << routingKey << "] from queue " << queue->getName()); - - if (propagate) - propagateFedOp(routingKey, string(), fedOpUnbind, string()); + QPID_LOG(debug, "Unbound key [" << routingKey << "] from queue " << queue->getName() + << " on exchange " << getName()); return true; } -bool TopicExchange::isBound(Queue::shared_ptr queue, const string& pattern) +/** returns a pointer to the BindingKey if the given queue is bound to this + * exchange using the routing pattern. 0 if queue binding does not exist. + */ +TopicExchange::BindingKey *TopicExchange::getQueueBinding(Queue::shared_ptr queue, const string& pattern) { // Note well: lock held by caller.... BindingKey *bk = bindingTree.getBindingKey(pattern); // Exact match against binding pattern - if (!bk) return false; + if (!bk) return 0; Binding::vector& qv(bk->bindingVector); Binding::vector::iterator q; for (q = qv.begin(); q != qv.end(); q++) if ((*q)->queue == queue) break; - return q != qv.end(); + return (q != qv.end()) ? bk : 0; } void TopicExchange::route(Deliverable& msg, const string& routingKey, const FieldTable* /*args*/) @@ -348,7 +365,7 @@ bool TopicExchange::isBound(Queue::shared_ptr queue, const string* const routing RWlock::ScopedRlock l(lock); if (routingKey && queue) { string key(normalize(*routingKey)); - return isBound(queue, key); + return getQueueBinding(queue, key) != 0; } else if (!routingKey && !queue) { return nBindings > 0; } else if (routingKey) { diff --git a/qpid/cpp/src/qpid/broker/TopicExchange.h b/qpid/cpp/src/qpid/broker/TopicExchange.h index a6c457d..0625889 100644 --- a/qpid/cpp/src/qpid/broker/TopicExchange.h +++ b/qpid/cpp/src/qpid/broker/TopicExchange.h @@ -136,7 +136,10 @@ class TopicExchange : public virtual Exchange { unsigned long nBindings; qpid::sys::RWlock lock; // protects bindingTree and nBindings - bool isBound(Queue::shared_ptr queue, const std::string& pattern); + BindingKey *getQueueBinding(Queue::shared_ptr queue, const std::string& pattern); + bool deleteBinding(Queue::shared_ptr queue, + const std::string& routingKey, + BindingKey *bk); class ReOriginIter; class BindingsFinderIter; diff --git a/qpid/cpp/src/qpid/broker/TxPublish.cpp b/qpid/cpp/src/qpid/broker/TxPublish.cpp index 36a451e..9c2cf4a 100644 --- a/qpid/cpp/src/qpid/broker/TxPublish.cpp +++ b/qpid/cpp/src/qpid/broker/TxPublish.cpp @@ -90,14 +90,7 @@ void TxPublish::deliverTo(const boost::shared_ptr<Queue>& queue){ void TxPublish::prepare(TransactionContext* ctxt, const boost::shared_ptr<Queue> queue) { - if (!queue->enqueue(ctxt, msg)){ - /** - * if not store then mark message for ack and deleivery once - * commit happens, as async IO will never set it when no store - * exists - */ - msg->enqueueComplete(); - } + queue->enqueue(ctxt, msg); } TxPublish::Commit::Commit(intrusive_ptr<Message>& _msg) : msg(_msg){} diff --git a/qpid/cpp/src/qpid/cluster/Cluster.cpp b/qpid/cpp/src/qpid/cluster/Cluster.cpp index a8a99d8..ee9f178 100644 --- a/qpid/cpp/src/qpid/cluster/Cluster.cpp +++ b/qpid/cpp/src/qpid/cluster/Cluster.cpp @@ -198,7 +198,7 @@ namespace _qmf = ::qmf::org::apache::qpid::cluster; * Currently use SVN revision to avoid clashes with versions from * different branches. */ -const uint32_t Cluster::CLUSTER_VERSION = 1058747; +const uint32_t Cluster::CLUSTER_VERSION = 1097431; struct ClusterDispatcher : public framing::AMQP_AllOperations::ClusterHandler { qpid::cluster::Cluster& cluster; @@ -235,7 +235,7 @@ struct ClusterDispatcher : public framing::AMQP_AllOperations::ClusterHandler { cluster.errorCheck(member, type, frameSeq, l); } void timerWakeup(const std::string& name) { cluster.timerWakeup(member, name, l); } - void timerDrop(const std::string& name) { cluster.timerWakeup(member, name, l); } + void timerDrop(const std::string& name) { cluster.timerDrop(member, name, l); } void shutdown(const Uuid& id) { cluster.shutdown(member, id, l); } void deliverToQueue(const std::string& queue, const std::string& message) { cluster.deliverToQueue(queue, message, l); diff --git a/qpid/cpp/src/qpid/cluster/Connection.cpp b/qpid/cpp/src/qpid/cluster/Connection.cpp index c1304c2..a75d135 100644 --- a/qpid/cpp/src/qpid/cluster/Connection.cpp +++ b/qpid/cpp/src/qpid/cluster/Connection.cpp @@ -35,6 +35,7 @@ #include "qpid/broker/Fairshare.h" #include "qpid/broker/Link.h" #include "qpid/broker/Bridge.h" +#include "qpid/broker/StatefulQueueObserver.h" #include "qpid/broker/Queue.h" #include "qpid/framing/enum.h" #include "qpid/framing/AMQFrame.h" @@ -556,6 +557,48 @@ void Connection::queueFairshareState(const std::string& qname, const uint8_t pri } } + +namespace { + // find a StatefulQueueObserver that matches a given identifier + class ObserverFinder { + const std::string id; + boost::shared_ptr<broker::QueueObserver> target; + ObserverFinder(const ObserverFinder&) {} + public: + ObserverFinder(const std::string& _id) : id(_id) {} + broker::StatefulQueueObserver *getObserver() + { + if (target) + return dynamic_cast<broker::StatefulQueueObserver *>(target.get()); + return 0; + } + void operator() (boost::shared_ptr<broker::QueueObserver> o) + { + if (!target) { + broker::StatefulQueueObserver *p = dynamic_cast<broker::StatefulQueueObserver *>(o.get()); + if (p && p->getId() == id) { + target = o; + } + } + } + }; +} + + +void Connection::queueObserverState(const std::string& qname, const std::string& observerId, const FieldTable& state) +{ + boost::shared_ptr<broker::Queue> queue(findQueue(qname)); + ObserverFinder finder(observerId); // find this observer + queue->eachObserver<ObserverFinder &>(finder); + broker::StatefulQueueObserver *so = finder.getObserver(); + if (so) { + so->setState( state ); + QPID_LOG(debug, "updated queue observer " << observerId << "'s state on queue " << qname << "; ..."); + return; + } + QPID_LOG(error, "Failed to find observer " << observerId << " state on queue " << qname << "; this will result in inconsistencies."); +} + void Connection::expiryId(uint64_t id) { cluster.getExpiryPolicy().setId(id); } diff --git a/qpid/cpp/src/qpid/cluster/Connection.h b/qpid/cpp/src/qpid/cluster/Connection.h index 8a9891d..83f925e 100644 --- a/qpid/cpp/src/qpid/cluster/Connection.h +++ b/qpid/cpp/src/qpid/cluster/Connection.h @@ -153,6 +153,7 @@ class Connection : void queuePosition(const std::string&, const framing::SequenceNumber&); void queueFairshareState(const std::string&, const uint8_t priority, const uint8_t count); + void queueObserverState(const std::string&, const std::string&, const framing::FieldTable&); void expiryId(uint64_t); void txStart(); diff --git a/qpid/cpp/src/qpid/cluster/UpdateClient.cpp b/qpid/cpp/src/qpid/cluster/UpdateClient.cpp index 8f751ad..a15c14f 100644 --- a/qpid/cpp/src/qpid/cluster/UpdateClient.cpp +++ b/qpid/cpp/src/qpid/cluster/UpdateClient.cpp @@ -49,6 +49,7 @@ #include "qpid/broker/TxPublish.h" #include "qpid/broker/RecoveredDequeue.h" #include "qpid/broker/RecoveredEnqueue.h" +#include "qpid/broker/StatefulQueueObserver.h" #include "qpid/framing/MessageTransferBody.h" #include "qpid/framing/ClusterConnectionMembershipBody.h" #include "qpid/framing/ClusterConnectionShadowReadyBody.h" @@ -167,6 +168,9 @@ void UpdateClient::update() { boost::bind(&UpdateClient::updateConnection, this, _1)); session.queueDelete(arg::queue=UPDATE); + // some Queue Observers need session state & msgs synced first, so sync observers now + b.getQueues().eachQueue(boost::bind(&UpdateClient::updateQueueObservers, this, _1)); + // Update queue listeners: must come after sessions so consumerNumbering is populated b.getQueues().eachQueue(boost::bind(&UpdateClient::updateQueueListeners, this, _1)); @@ -615,4 +619,23 @@ void UpdateClient::updateBridge(const boost::shared_ptr<broker::Bridge>& bridge) ClusterConnectionProxy(session).config(encode(*bridge)); } +void UpdateClient::updateQueueObservers(const boost::shared_ptr<broker::Queue>& q) +{ + q->eachObserver(boost::bind(&UpdateClient::updateObserver, this, q, _1)); +} + +void UpdateClient::updateObserver(const boost::shared_ptr<broker::Queue>& q, + boost::shared_ptr<broker::QueueObserver> o) +{ + qpid::framing::FieldTable state; + broker::StatefulQueueObserver *so = dynamic_cast<broker::StatefulQueueObserver *>(o.get()); + if (so) { + so->getState( state ); + std::string id(so->getId()); + QPID_LOG(debug, *this << " updating queue " << q->getName() << "'s observer " << id); + ClusterConnectionProxy(session).queueObserverState( q->getName(), id, state ); + } +} + + }} // namespace qpid::cluster diff --git a/qpid/cpp/src/qpid/cluster/UpdateClient.h b/qpid/cpp/src/qpid/cluster/UpdateClient.h index 7520bb8..bbf7a94 100644 --- a/qpid/cpp/src/qpid/cluster/UpdateClient.h +++ b/qpid/cpp/src/qpid/cluster/UpdateClient.h @@ -51,6 +51,7 @@ class SemanticState; class Decoder; class Link; class Bridge; +class QueueObserver; } // namespace broker @@ -104,6 +105,8 @@ class UpdateClient : public sys::Runnable { void updateLinks(); void updateLink(const boost::shared_ptr<broker::Link>&); void updateBridge(const boost::shared_ptr<broker::Bridge>&); + void updateQueueObservers(const boost::shared_ptr<broker::Queue>&); + void updateObserver(const boost::shared_ptr<broker::Queue>&, boost::shared_ptr<broker::QueueObserver>); Numbering<broker::SemanticState::ConsumerImpl*> consumerNumbering; diff --git a/qpid/cpp/src/qpid/log/Logger.cpp b/qpid/cpp/src/qpid/log/Logger.cpp index 2217cdd..2339a62 100644 --- a/qpid/cpp/src/qpid/log/Logger.cpp +++ b/qpid/cpp/src/qpid/log/Logger.cpp @@ -22,6 +22,7 @@ #include "qpid/memory.h" #include "qpid/sys/Thread.h" #include "qpid/sys/Time.h" +#include "qpid/DisableExceptionLogging.h" #include <boost/pool/detail/singleton.hpp> #include <boost/bind.hpp> #include <boost/function.hpp> @@ -48,11 +49,16 @@ Logger& Logger::instance() { } Logger::Logger() : flags(0) { + // Disable automatic logging in Exception constructors to avoid + // re-entrant use of logger singleton if there is an error in + // option parsing. + DisableExceptionLogging del; + // Initialize myself from env variables so all programs // (e.g. tests) can use logging even if they don't parse // command line args. Options opts(""); - opts.parse(0, 0); + opts.parse(0, 0); configure(opts); } @@ -73,7 +79,7 @@ void Logger::log(const Statement& s, const std::string& msg) { std::ostringstream os; if (!prefix.empty()) os << prefix << ": "; - if (flags&TIME) + if (flags&TIME) qpid::sys::outputFormattedNow(os); if (flags&LEVEL) os << LevelTraits::name(s.level) << " "; @@ -140,7 +146,7 @@ void Logger::configure(const Options& opts) { Options o(opts); if (o.trace) o.selectors.push_back("trace+"); - format(o); + format(o); select(Selector(o)); setPrefix(opts.prefix); options.sinkOptions->setup(this); diff --git a/qpid/cpp/src/qpid/sys/Timer.cpp b/qpid/cpp/src/qpid/sys/Timer.cpp index 85e4108..fdb2e8c 100644 --- a/qpid/cpp/src/qpid/sys/Timer.cpp +++ b/qpid/cpp/src/qpid/sys/Timer.cpp @@ -185,7 +185,11 @@ void Timer::stop() // Allow subclasses to override behavior when firing a task. void Timer::fire(boost::intrusive_ptr<TimerTask> t) { - t->fireTask(); + try { + t->fireTask(); + } catch (const std::exception& e) { + QPID_LOG(error, "Exception thrown by timer task " << t->getName() << ": " << e.what()); + } } // Provided for subclasses: called when a task is droped. diff --git a/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp b/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp index c80c94c..af58b7d 100644 --- a/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp +++ b/qpid/cpp/src/qpid/sys/rdma/RdmaIO.cpp @@ -141,7 +141,7 @@ namespace Rdma { qp->allocateRecvBuffers(recvBufferCount, bufferSize+FrameHeaderSize); // Create xmit buffers - qp->createSendBuffers(xmitBufferCount, bufferSize+FrameHeaderSize); + qp->createSendBuffers(xmitBufferCount, bufferSize, FrameHeaderSize); } AsynchIO::~AsynchIO() { diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp index 6d38c42..efe454c 100644 --- a/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.cpp @@ -50,8 +50,9 @@ namespace Rdma { return count; } - Buffer::Buffer(uint32_t lkey, char* bytes, const int32_t byteCount) : - bufferSize(byteCount) + Buffer::Buffer(uint32_t lkey, char* bytes, const int32_t byteCount, + const int32_t reserve) : + bufferSize(byteCount + reserve), reserved(reserve) { sge.addr = (uintptr_t) bytes; sge.length = 0; @@ -163,21 +164,21 @@ namespace Rdma { } // Create buffers to use for writing - void QueuePair::createSendBuffers(int sendBufferCount, int bufferSize) + void QueuePair::createSendBuffers(int sendBufferCount, int bufferSize, int reserved) { assert(!smr); // Round up buffersize to cacheline (64 bytes) - bufferSize = (bufferSize+63) & (~63); + int dataLength = (bufferSize+reserved+63) & (~63); // Allocate memory block for all receive buffers - char* mem = new char [sendBufferCount * bufferSize]; - smr = regMr(pd.get(), mem, sendBufferCount * bufferSize, ::IBV_ACCESS_LOCAL_WRITE); + char* mem = new char [sendBufferCount * dataLength]; + smr = regMr(pd.get(), mem, sendBufferCount * dataLength, ::IBV_ACCESS_LOCAL_WRITE); sendBuffers.reserve(sendBufferCount); freeBuffers.reserve(sendBufferCount); for (int i = 0; i<sendBufferCount; ++i) { // Allocate xmit buffer - sendBuffers.push_back(Buffer(smr->lkey, &mem[i*bufferSize], bufferSize)); + sendBuffers.push_back(Buffer(smr->lkey, &mem[i*dataLength], bufferSize, reserved)); freeBuffers.push_back(i); } } diff --git a/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h index 28bddd2..3737664 100644 --- a/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h +++ b/qpid/cpp/src/qpid/sys/rdma/rdma_wrap.h @@ -57,8 +57,9 @@ namespace Rdma { void dataCount(int32_t); private: - Buffer(uint32_t lkey, char* bytes, const int32_t byteCount); + Buffer(uint32_t lkey, char* bytes, const int32_t byteCount, const int32_t reserve=0); int32_t bufferSize; + int32_t reserved; // for framing header ::ibv_sge sge; }; @@ -66,8 +67,9 @@ namespace Rdma { return (char*) sge.addr; } + /** return the number of bytes available for application data */ inline int32_t Buffer::byteCount() const { - return bufferSize; + return bufferSize - reserved; } inline int32_t Buffer::dataCount() const { @@ -136,7 +138,7 @@ namespace Rdma { typedef boost::intrusive_ptr<QueuePair> intrusive_ptr; // Create a buffers to use for writing - void createSendBuffers(int sendBufferCount, int bufferSize); + void createSendBuffers(int sendBufferCount, int dataSize, int headerSize); // Get a send buffer Buffer* getSendBuffer(); diff --git a/qpid/cpp/src/qpid/types/Variant.cpp b/qpid/cpp/src/qpid/types/Variant.cpp index 9cc3cfe..30d7ff1 100644 --- a/qpid/cpp/src/qpid/types/Variant.cpp +++ b/qpid/cpp/src/qpid/types/Variant.cpp @@ -781,6 +781,10 @@ Variant& Variant::parse(const std::string& s) return *this; } +Variant& Variant::fromString(const std::string& s) +{ + return parse(s); +} VariantType Variant::getType() const { return impl ? impl->getType() : VAR_VOID; } bool Variant::isVoid() const { return getType() == VAR_VOID; } diff --git a/qpid/cpp/src/tests/QueueFlowLimitTest.cpp b/qpid/cpp/src/tests/QueueFlowLimitTest.cpp index 7018409..8a6923f 100644 --- a/qpid/cpp/src/tests/QueueFlowLimitTest.cpp +++ b/qpid/cpp/src/tests/QueueFlowLimitTest.cpp @@ -75,8 +75,10 @@ public: QueuedMessage createMessage(uint32_t size) { + static uint32_t seqNum; QueuedMessage msg; msg.payload = MessageUtils::createMessage(); + msg.position = ++seqNum; MessageUtils::addContent(msg.payload, std::string (size, 'x')); return msg; } diff --git a/qpid/cpp/src/tests/QueuePolicyTest.cpp b/qpid/cpp/src/tests/QueuePolicyTest.cpp index f9c058c..5455105 100644 --- a/qpid/cpp/src/tests/QueuePolicyTest.cpp +++ b/qpid/cpp/src/tests/QueuePolicyTest.cpp @@ -386,6 +386,7 @@ QPID_AUTO_TEST_CASE(testCapacityConversion) { FieldTable args; args.setString("qpid.max_count", "5"); + args.setString("qpid.flow_stop_count", "0"); ProxySessionFixture f; std::string q("q"); diff --git a/qpid/cpp/src/tests/QueueTest.cpp b/qpid/cpp/src/tests/QueueTest.cpp index 2059727..34e4592 100644 --- a/qpid/cpp/src/tests/QueueTest.cpp +++ b/qpid/cpp/src/tests/QueueTest.cpp @@ -303,11 +303,11 @@ QPID_AUTO_TEST_CASE(testSeek){ QueuedMessage qm; queue->dispatch(consumer); - + BOOST_CHECK_EQUAL(msg3.get(), consumer->last.get()); queue->dispatch(consumer); queue->dispatch(consumer); // make sure over-run is safe - + } QPID_AUTO_TEST_CASE(testSearch){ @@ -325,15 +325,15 @@ QPID_AUTO_TEST_CASE(testSearch){ SequenceNumber seq(2); QueuedMessage qm = queue->find(seq); - + BOOST_CHECK_EQUAL(seq.getValue(), qm.position.getValue()); - + queue->acquire(qm); BOOST_CHECK_EQUAL(queue->getMessageCount(), 2u); SequenceNumber seq1(3); QueuedMessage qm1 = queue->find(seq1); BOOST_CHECK_EQUAL(seq1.getValue(), qm1.position.getValue()); - + } const std::string nullxid = ""; @@ -875,28 +875,40 @@ QPID_AUTO_TEST_CASE(testFlowToDiskBlocking){ intrusive_ptr<Message> msg02 = mkMsg(testStore, std::string(5, 'X')); // transient w/ content DeliverableMessage dmsg02(msg02); - BOOST_CHECK_THROW(sbtFanout1.route(dmsg02, "", 0), ResourceLimitExceededException); + { + ScopedSuppressLogging sl; // suppress expected error messages. + BOOST_CHECK_THROW(sbtFanout1.route(dmsg02, "", 0), ResourceLimitExceededException); + } msg02->tryReleaseContent(); BOOST_CHECK_EQUAL(msg02->isContentReleased(), false); BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); intrusive_ptr<Message> msg03 = mkMsg(testStore, std::string(5, 'X'), true); // durable w/ content DeliverableMessage dmsg03(msg03); - BOOST_CHECK_THROW(sbtFanout1.route(dmsg03, "", 0), ResourceLimitExceededException); + { + ScopedSuppressLogging sl; // suppress expected error messages. + BOOST_CHECK_THROW(sbtFanout1.route(dmsg03, "", 0), ResourceLimitExceededException); + } msg03->tryReleaseContent(); BOOST_CHECK_EQUAL(msg03->isContentReleased(), false); BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); intrusive_ptr<Message> msg04 = mkMsg(testStore); // transient no content DeliverableMessage dmsg04(msg04); - BOOST_CHECK_THROW(sbtFanout1.route(dmsg04, "", 0), ResourceLimitExceededException); + { + ScopedSuppressLogging sl; // suppress expected error messages. + BOOST_CHECK_THROW(sbtFanout1.route(dmsg04, "", 0), ResourceLimitExceededException); + } msg04->tryReleaseContent(); BOOST_CHECK_EQUAL(msg04->isContentReleased(), false); BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); intrusive_ptr<Message> msg05 = mkMsg(testStore, "", true); // durable no content DeliverableMessage dmsg05(msg05); - BOOST_CHECK_THROW(sbtFanout1.route(dmsg05, "", 0), ResourceLimitExceededException); + { + ScopedSuppressLogging sl; // suppress expected error messages. + BOOST_CHECK_THROW(sbtFanout1.route(dmsg05, "", 0), ResourceLimitExceededException); + } msg05->tryReleaseContent(); BOOST_CHECK_EQUAL(msg05->isContentReleased(), false); BOOST_CHECK_EQUAL(1u, tq1->getMessageCount()); diff --git a/qpid/cpp/src/tests/brokertest.py b/qpid/cpp/src/tests/brokertest.py index 19e97ce..ab16f0e 100644 --- a/qpid/cpp/src/tests/brokertest.py +++ b/qpid/cpp/src/tests/brokertest.py @@ -530,18 +530,24 @@ class BrokerTest(TestCase): cluster = Cluster(self, count, args, expect=expect, wait=wait) return cluster - def assert_browse(self, session, queue, expect_contents, timeout=0): + def browse(self, session, queue, timeout=0): """Assert that the contents of messages on queue (as retrieved using session and timeout) exactly match the strings in expect_contents""" - r = session.receiver("%s;{mode:browse}"%(queue)) - actual_contents = [] try: - for c in expect_contents: actual_contents.append(r.fetch(timeout=timeout).content) - while True: actual_contents.append(r.fetch(timeout=0).content) # Check for extra messages. - except messaging.Empty: pass - r.close() + contents = [] + try: + while True: contents.append(r.fetch(timeout=timeout).content) + except messaging.Empty: pass + finally: pass #FIXME aconway 2011-04-14: r.close() + return contents + + def assert_browse(self, session, queue, expect_contents, timeout=0): + """Assert that the contents of messages on queue (as retrieved + using session and timeout) exactly match the strings in + expect_contents""" + actual_contents = self.browse(session, queue, timeout) self.assertEqual(expect_contents, actual_contents) class RethrownException(Exception): diff --git a/qpid/cpp/src/tests/cluster_tests.py b/qpid/cpp/src/tests/cluster_tests.py index 3f19411..da2f47f 100755 --- a/qpid/cpp/src/tests/cluster_tests.py +++ b/qpid/cpp/src/tests/cluster_tests.py @@ -301,6 +301,13 @@ acl allow all all qpid_tool.wait() scanner.join() assert scanner.found + # Regression test for https://issues.apache.org/jira/browse/QPID-3235 + # Inconsistent stats when changing elder. + + # Force a change of elder + cluster0.start() + cluster0[0].kill() + time.sleep(2) # Allow a management interval to pass. # Verify logs are consistent cluster_test_logs.verify_logs() @@ -346,7 +353,7 @@ acl allow all all Thread.__init__(self) def run(self): try: - self.sender.send(self.msg) + self.sender.send(self.msg, sync=True) self.condition.acquire() try: self.blocked = False @@ -378,11 +385,12 @@ acl allow all all ssn0 = brokers.first().connect().session() s0 = ssn0.sender("flq; {create:always, node:{type:queue, x-declare:{arguments:{'qpid.flow_stop_count':5, 'qpid.flow_resume_count':3}}}}") brokers.first().startQmf() - q = [q for q in brokers.first().qmf_session.getObjects(_class="queue") if q.name == "flq"][0] - oid = q.getObjectId() - self.assertEqual(q.name, "flq") - self.assertEqual(q.arguments, {u'qpid.flow_stop_count': 5L, u'qpid.flow_resume_count': 3L}) - assert not q.flowStopped + q1 = [q for q in brokers.first().qmf_session.getObjects(_class="queue") if q.name == "flq"][0] + oid = q1.getObjectId() + self.assertEqual(q1.name, "flq") + self.assertEqual(q1.arguments, {u'qpid.flow_stop_count': 5L, u'qpid.flow_resume_count': 3L}) + assert not q1.flowStopped + self.assertEqual(q1.flowStoppedCount, 0) # fill the queue on one broker until flow control is active for x in range(5): s0.send(Message(str(x))) @@ -390,18 +398,20 @@ acl allow all all sender.start() # Tests that sender does block # Verify the broker queue goes into a flowStopped state deadline = time.time() + 1 - while not q.flowStopped and time.time() < deadline: q.update() - assert q.flowStopped + while not q1.flowStopped and time.time() < deadline: q1.update() + assert q1.flowStopped + self.assertEqual(q1.flowStoppedCount, 1) sender.assert_blocked() # Still blocked # Now verify the both brokers in cluster have same configuration brokers.second().startQmf() qs = brokers.second().qmf_session.getObjects(_objectId=oid) self.assertEqual(len(qs), 1) - q = qs[0] - self.assertEqual(q.name, "flq") - self.assertEqual(q.arguments, {u'qpid.flow_stop_count': 5L, u'qpid.flow_resume_count': 3L}) - assert q.flowStopped + q2 = qs[0] + self.assertEqual(q2.name, "flq") + self.assertEqual(q2.arguments, {u'qpid.flow_stop_count': 5L, u'qpid.flow_resume_count': 3L}) + assert q2.flowStopped + self.assertEqual(q2.flowStoppedCount, 1) # now drain the queue using a session to the other broker ssn1 = brokers.second().connect().session() @@ -411,6 +421,12 @@ acl allow all all ssn1.acknowledge() sender.wait() # Verify no longer blocked. + # and re-verify state of queue on both brokers + q1.update() + assert not q1.flowStopped + q2.update() + assert not q2.flowStopped + ssn0.connection.close() ssn1.connection.close() cluster_test_logs.verify_logs() @@ -424,7 +440,6 @@ acl allow all all self.queue_flowlimit_test(Brokers()) def test_queue_flowlimit_cluster(self): - return # TODO aconway 2011-02-18: disabled till fixed, QPID-2935 cluster = self.cluster(2) class Brokers: def first(self): return cluster[0] @@ -432,7 +447,6 @@ acl allow all all self.queue_flowlimit_test(Brokers()) def test_queue_flowlimit_cluster_join(self): - return # TODO aconway 2011-02-18: disabled till fixed, QPID-2935 cluster = self.cluster(1) class Brokers: def first(self): return cluster[0] @@ -441,6 +455,103 @@ acl allow all all return cluster[1] self.queue_flowlimit_test(Brokers()) + def test_queue_flowlimit_replicate(self): + """ Verify that a queue which is in flow control BUT has drained BELOW + the flow control 'stop' threshold, is correctly replicated when a new + broker is added to the cluster. + """ + + class AsyncSender(Thread): + """Send a fixed number of msgs from a sender in a separate thread + so it may block without blocking the test. + """ + def __init__(self, broker, address, count=1, size=4): + Thread.__init__(self) + self.daemon = True + self.broker = broker + self.queue = address + self.count = count + self.size = size + self.done = False + + def run(self): + self.sender = subprocess.Popen(["qpid-send", + "--capacity=1", + "--content-size=%s" % self.size, + "--messages=%s" % self.count, + "--failover-updates", + "--connection-options={reconnect:true}", + "--address=%s" % self.queue, + "--broker=%s" % self.broker.host_port()]) + self.sender.wait() + self.done = True + + cluster = self.cluster(2) + # create a queue with rather draconian flow control settings + ssn0 = cluster[0].connect().session() + s0 = ssn0.sender("flq; {create:always, node:{type:queue, x-declare:{arguments:{'qpid.flow_stop_count':100, 'qpid.flow_resume_count':20}}}}") + + # fire off the sending thread to broker[0], and wait until the queue + # hits flow control on broker[1] + sender = AsyncSender(cluster[0], "flq", count=110); + sender.start(); + + cluster[1].startQmf() + q_obj = [q for q in cluster[1].qmf_session.getObjects(_class="queue") if q.name == "flq"][0] + deadline = time.time() + 10 + while not q_obj.flowStopped and time.time() < deadline: + q_obj.update() + assert q_obj.flowStopped + assert not sender.done + assert q_obj.msgDepth < 110 + + # Now drain enough messages on broker[1] to drop below the flow stop + # threshold, but not relieve flow control... + receiver = subprocess.Popen(["qpid-receive", + "--messages=15", + "--timeout=1", + "--print-content=no", + "--failover-updates", + "--connection-options={reconnect:true}", + "--ack-frequency=1", + "--address=flq", + "--broker=%s" % cluster[1].host_port()]) + receiver.wait() + q_obj.update() + assert q_obj.flowStopped + assert not sender.done + current_depth = q_obj.msgDepth + + # add a new broker to the cluster, and verify that the queue is in flow + # control on that broker + cluster.start() + cluster[2].startQmf() + q_obj = [q for q in cluster[2].qmf_session.getObjects(_class="queue") if q.name == "flq"][0] + assert q_obj.flowStopped + assert q_obj.msgDepth == current_depth + + # now drain the queue on broker[2], and verify that the sender becomes + # unblocked + receiver = subprocess.Popen(["qpid-receive", + "--messages=95", + "--timeout=1", + "--print-content=no", + "--failover-updates", + "--connection-options={reconnect:true}", + "--ack-frequency=1", + "--address=flq", + "--broker=%s" % cluster[2].host_port()]) + receiver.wait() + q_obj.update() + assert not q_obj.flowStopped + assert q_obj.msgDepth == 0 + + # verify that the sender has become unblocked + sender.join(timeout=5) + assert not sender.isAlive() + assert sender.done + + def test_alternate_exchange_update(self): """Verify that alternate-exchange on exchanges and queues is propagated to new members of a cluster. """ cluster = self.cluster(1) @@ -468,6 +579,83 @@ acl allow all all cluster.start() verify(cluster[1]) + def test_binding_order(self): + """Regression test for binding order inconsistency in cluster""" + cluster = self.cluster(1) + c0 = cluster[0].connect() + s0 = c0.session() + # Declare multiple queues bound to same key on amq.topic + def declare(q,max=0): + if max: declare = 'x-declare:{arguments:{"qpid.max_count":%d, "qpid.flow_stop_count":0}}'%max + else: declare = 'x-declare:{}' + bind='x-bindings:[{queue:%s,key:key,exchange:"amq.topic"}]'%(q) + s0.sender("%s;{create:always,node:{%s,%s}}" % (q,declare,bind)) + declare('d',max=4) # Only one with a limit + for q in ['c', 'b','a']: declare(q) + # Add a cluster member, send enough messages to exceed the max count + cluster.start() + try: + s = s0.sender('amq.topic/key') + for m in xrange(1,6): s.send(Message(str(m))) + self.fail("Expected capacity exceeded exception") + except messaging.exceptions.TargetCapacityExceeded: pass + c1 = cluster[1].connect() + s1 = c1.session() + s0 = c0.session() # Old session s0 is broken by exception. + # Verify queue contents are consistent. + for q in ['a','b','c','d']: + self.assertEqual(self.browse(s0, q), self.browse(s1, q)) + # Verify queue contents are "best effort" + for q in ['a','b','c']: self.assert_browse(s1,q,[str(n) for n in xrange(1,6)]) + self.assert_browse(s1,'d',[str(n) for n in xrange(1,5)]) + + def test_deleted_exchange(self): + """QPID-3215: cached exchange reference can cause cluster inconsistencies + if exchange is deleted/recreated + Verify stand-alone case + """ + cluster = self.cluster() + # Verify we do not route message via an exchange that has been destroyed. + cluster.start() + s0 = cluster[0].connect().session() + self.evaluate_address(s0, "ex;{create:always,node:{type:topic}}") + self.evaluate_address(s0, "q;{create:always,node:{x-bindings:[{exchange:'ex',queue:q,key:foo}]}}") + send0 = s0.sender("ex/foo") + send0.send("foo") + self.assert_browse(s0, "q", ["foo"]) + self.evaluate_address(s0, "ex;{delete:always}") + try: + send0.send("bar") # Should fail, exchange is deleted. + self.fail("Expected not-found exception") + except qpid.messaging.NotFound: pass + self.assert_browse(cluster[0].connect().session(), "q", ["foo"]) + + def test_deleted_exchange_inconsistent(self): + """QPID-3215: cached exchange reference can cause cluster inconsistencies + if exchange is deleted/recreated + + Verify cluster inconsistency. + """ + cluster = self.cluster() + cluster.start() + s0 = cluster[0].connect().session() + self.evaluate_address(s0, "ex;{create:always,node:{type:topic}}") + self.evaluate_address(s0, "q;{create:always,node:{x-bindings:[{exchange:'ex',queue:q,key:foo}]}}") + send0 = s0.sender("ex/foo") + send0.send("foo") + self.assert_browse(s0, "q", ["foo"]) + + cluster.start() + s1 = cluster[1].connect().session() + self.evaluate_address(s0, "ex;{delete:always}") + try: + send0.send("bar") + self.fail("Expected not-found exception") + except qpid.messaging.NotFound: pass + + self.assert_browse(s1, "q", ["foo"]) + + class LongTests(BrokerTest): """Tests that can run for a long time if -DDURATION=<minutes> is set""" def duration(self): @@ -629,6 +817,41 @@ class LongTests(BrokerTest): for i in xrange(1000): cluster[0].connect().close() cluster_test_logs.verify_logs() + def test_flowlimit_failover(self): + """Test fail-over during continuous send-receive with flow control + active. + """ + + # Original cluster will all be killed so expect exit with failure + cluster = self.cluster(3, expect=EXPECT_EXIT_FAIL) + #for b in cluster: ErrorGenerator(b) + + # create a queue with rather draconian flow control settings + ssn0 = cluster[0].connect().session() + s0 = ssn0.sender("test-queue; {create:always, node:{type:queue, x-declare:{arguments:{'qpid.flow_stop_count':2000, 'qpid.flow_resume_count':100}}}}") + + receiver = NumberedReceiver(cluster[2]) + receiver.start() + senders = [NumberedSender(cluster[i]) for i in range(1,3)] + for s in senders: + s.start() + + # Kill original brokers, start new ones for the duration. + endtime = time.time() + self.duration(); + i = 0 + while time.time() < endtime: + cluster[i].kill() + i += 1 + b = cluster.start(expect=EXPECT_EXIT_FAIL) + #ErrorGenerator(b) + time.sleep(5) + #b = cluster[0] + #b.startQmf() + for s in senders: + s.stop() + receiver.stop() + for i in range(i, len(cluster)): cluster[i].kill() + class StoreTests(BrokerTest): """ diff --git a/qpid/cpp/src/tests/federation.py b/qpid/cpp/src/tests/federation.py index 973a1d3..201b06a 100755 --- a/qpid/cpp/src/tests/federation.py +++ b/qpid/cpp/src/tests/federation.py @@ -23,7 +23,7 @@ from qpid.testlib import TestBase010 from qpid.datatypes import Message from qpid.queue import Empty from qpid.util import URL -from time import sleep +from time import sleep, time class _FedBroker(object): @@ -1791,3 +1791,301 @@ class FederationTests(TestBase010): if headers: return headers[name] return None + + def test_dynamic_topic_bounce(self): + """ Bounce the connection between federated Topic Exchanges. + """ + class Params: + def exchange_type(self): return "topic" + def bind_queue(self, ssn, qname, ename): + ssn.exchange_bind(queue=qname, exchange=ename, + binding_key="spud.*") + def unbind_queue(self, ssn, qname, ename): + ssn.exchange_unbind(queue=qname, exchange=ename, binding_key="spud.*") + def delivery_properties(self, ssn): + return ssn.delivery_properties(routing_key="spud.boy") + + self.generic_dynamic_bounce_test(Params()) + + def test_dynamic_direct_bounce(self): + """ Bounce the connection between federated Direct Exchanges. + """ + class Params: + def exchange_type(self): return "direct" + def bind_queue(self, ssn, qname, ename): + ssn.exchange_bind(queue=qname, exchange=ename, binding_key="spud") + def unbind_queue(self, ssn, qname, ename): + ssn.exchange_unbind(queue=qname, exchange=ename, binding_key="spud") + def delivery_properties(self, ssn): + return ssn.delivery_properties(routing_key="spud") + self.generic_dynamic_bounce_test(Params()) + + def test_dynamic_fanout_bounce(self): + """ Bounce the connection between federated Fanout Exchanges. + """ + class Params: + def exchange_type(self): return "fanout" + def bind_queue(self, ssn, qname, ename): + ssn.exchange_bind(queue=qname, exchange=ename) + def unbind_queue(self, ssn, qname, ename): + ssn.exchange_unbind(queue=qname, exchange=ename) + def delivery_properties(self, ssn): + return ssn.delivery_properties(routing_key="spud") + self.generic_dynamic_bounce_test(Params()) + + def test_dynamic_headers_bounce(self): + """ Bounce the connection between federated Headers Exchanges. + """ + class Params: + def exchange_type(self): return "headers" + def bind_queue(self, ssn, qname, ename): + ssn.exchange_bind(queue=qname, exchange=ename, + binding_key="spud", arguments={'x-match':'any', 'class':'first'}) + def unbind_queue(self, ssn, qname, ename): + ssn.exchange_unbind(queue=qname, exchange=ename, binding_key="spud") + def delivery_properties(self, ssn): + return ssn.message_properties(application_headers={'class':'first'}) + ## @todo KAG - re-enable once federation bugs with headers exchanges + ## are fixed. + #self.generic_dynamic_bounce_test(Params()) + return + + + def generic_dynamic_bounce_test(self, params): + """ Verify that a federated broker can maintain a binding to a local + queue using the same key as a remote binding. Destroy and reconnect + the federation link, and verify routes are restored correctly. + See QPID-3170. + Topology: + + Queue1 <---"Key"---B0<==[Federated Exchange]==>B1---"Key"--->Queue2 + """ + session = self.session + + # create the federation + + self.startQmf() + qmf = self.qmf + + self._setup_brokers() + + # create exchange on each broker, and retrieve the corresponding + # management object for that exchange + + exchanges=[] + for _b in self._brokers[0:2]: + _b.client_session.exchange_declare(exchange="fedX", type=params.exchange_type()) + self.assertEqual(_b.client_session.exchange_query(name="fedX").type, + params.exchange_type(), "exchange_declare failed!") + # pull the exchange out of qmf... + retries = 0 + my_exchange = None + timeout = time() + 10 + while my_exchange is None and time() <= timeout: + objs = qmf.getObjects(_broker=_b.qmf_broker, _class="exchange") + for ooo in objs: + if ooo.name == "fedX": + my_exchange = ooo + break + if my_exchange is None: + self.fail("QMF failed to find new exchange!") + exchanges.append(my_exchange) + + # + # on each broker, create a local queue bound to the exchange with the + # same key value. + # + + self._brokers[0].client_session.queue_declare(queue="fedX1", exclusive=True, auto_delete=True) + params.bind_queue(self._brokers[0].client_session, "fedX1", "fedX") + self.subscribe(self._brokers[0].client_session, queue="fedX1", destination="f1") + queue_0 = self._brokers[0].client_session.incoming("f1") + + self._brokers[1].client_session.queue_declare(queue="fedX1", exclusive=True, auto_delete=True) + params.bind_queue(self._brokers[1].client_session, "fedX1", "fedX") + self.subscribe(self._brokers[1].client_session, queue="fedX1", destination="f1") + queue_1 = self._brokers[1].client_session.incoming("f1") + + # now federate the two brokers + + # connect B0 --> B1 + result = self._brokers[1].qmf_object.connect(self._brokers[0].host, + self._brokers[0].port, + False, "PLAIN", "guest", "guest", "tcp") + self.assertEqual(result.status, 0) + + # connect B1 --> B0 + result = self._brokers[0].qmf_object.connect(self._brokers[1].host, + self._brokers[1].port, + False, "PLAIN", "guest", "guest", "tcp") + self.assertEqual(result.status, 0) + + # for each link, bridge the "fedX" exchanges: + + for _l in qmf.getObjects(_class="link"): + # print("Link=%s:%s %s" % (_l.host, _l.port, str(_l.getBroker()))) + result = _l.bridge(False, # durable + "fedX", # src + "fedX", # dst + "", # key + "", # tag + "", # excludes + False, # srcIsQueue + False, # srcIsLocal + True, # dynamic + 0) # sync + self.assertEqual(result.status, 0) + + # wait for all the inter-broker links to become operational + operational = False + timeout = time() + 10 + while not operational and time() <= timeout: + operational = True + for _l in qmf.getObjects(_class="link"): + #print("Link=%s:%s %s" % (_l.host, _l.port, str(_l.state))) + if _l.state != "Operational": + operational = False + self.failUnless(operational, "inter-broker links failed to become operational.") + + # @todo - There is no way to determine when the bridge objects become + # active. + + # wait until the binding key has propagated to each broker - each + # broker should see 2 bindings (1 local, 1 remote) + + binding_counts = [2, 2] + self.assertEqual(len(binding_counts), len(exchanges), "Update Test!") + for i in range(2): + exchanges[i].update() + timeout = time() + 10 + while exchanges[i].bindingCount < binding_counts[i] and time() <= timeout: + exchanges[i].update() + self.failUnless(exchanges[i].bindingCount == binding_counts[i]) + + # send 10 msgs to B0 + for i in range(1, 11): + # dp = self._brokers[0].client_session.delivery_properties(routing_key=params.routing_key()) + dp = params.delivery_properties(self._brokers[0].client_session) + self._brokers[0].client_session.message_transfer(destination="fedX", message=Message(dp, "Message_trp %d" % i)) + + # get exactly 10 msgs on B0's local queue and B1's queue + for i in range(1, 11): + try: + msg = queue_0.get(timeout=5) + self.assertEqual("Message_trp %d" % i, msg.body) + msg = queue_1.get(timeout=5) + self.assertEqual("Message_trp %d" % i, msg.body) + except Empty: + self.fail("Only got %d msgs - expected 10" % i) + try: + extra = queue_0.get(timeout=1) + self.fail("Got unexpected message in queue_0: " + extra.body) + except Empty: None + + try: + extra = queue_1.get(timeout=1) + self.fail("Got unexpected message in queue_1: " + extra.body) + except Empty: None + + # + # Tear down the bridges between the two exchanges, then wait + # for the bindings to be cleaned up + # + + for _b in qmf.getObjects(_class="bridge"): + result = _b.close() + self.assertEqual(result.status, 0) + + binding_counts = [1, 1] + self.assertEqual(len(binding_counts), len(exchanges), "Update Test!") + for i in range(2): + exchanges[i].update() + timeout = time() + 10 + while exchanges[i].bindingCount != binding_counts[i] and time() <= timeout: + exchanges[i].update() + self.failUnless(exchanges[i].bindingCount == binding_counts[i]) + + # + # restore the bridges between the two exchanges, and wait for the + # bindings to propagate. + # + + for _l in qmf.getObjects(_class="link"): + # print("Link=%s:%s %s" % (_l.host, _l.port, str(_l.getBroker()))) + result = _l.bridge(False, # durable + "fedX", # src + "fedX", # dst + "", # key + "", # tag + "", # excludes + False, # srcIsQueue + False, # srcIsLocal + True, # dynamic + 0) # sync + self.assertEqual(result.status, 0) + + binding_counts = [2, 2] + self.assertEqual(len(binding_counts), len(exchanges), "Update Test!") + for i in range(2): + exchanges[i].update() + timeout = time() + 10 + while exchanges[i].bindingCount != binding_counts[i] and time() <= timeout: + exchanges[i].update() + self.failUnless(exchanges[i].bindingCount == binding_counts[i]) + + # + # verify traffic flows correctly + # + + for i in range(1, 11): + #dp = self._brokers[1].client_session.delivery_properties(routing_key=params.routing_key()) + dp = params.delivery_properties(self._brokers[1].client_session) + self._brokers[1].client_session.message_transfer(destination="fedX", message=Message(dp, "Message_trp %d" % i)) + + # get exactly 10 msgs on B0's queue and B1's queue + for i in range(1, 11): + try: + msg = queue_0.get(timeout=5) + self.assertEqual("Message_trp %d" % i, msg.body) + msg = queue_1.get(timeout=5) + self.assertEqual("Message_trp %d" % i, msg.body) + except Empty: + self.fail("Only got %d msgs - expected 10" % i) + try: + extra = queue_0.get(timeout=1) + self.fail("Got unexpected message in queue_0: " + extra.body) + except Empty: None + + try: + extra = queue_1.get(timeout=1) + self.fail("Got unexpected message in queue_1: " + extra.body) + except Empty: None + + + # + # cleanup + # + params.unbind_queue(self._brokers[0].client_session, "fedX1", "fedX") + self._brokers[0].client_session.message_cancel(destination="f1") + self._brokers[0].client_session.queue_delete(queue="fedX1") + + params.unbind_queue(self._brokers[1].client_session, "fedX1", "fedX") + self._brokers[1].client_session.message_cancel(destination="f1") + self._brokers[1].client_session.queue_delete(queue="fedX1") + + for _b in qmf.getObjects(_class="bridge"): + result = _b.close() + self.assertEqual(result.status, 0) + + for _l in qmf.getObjects(_class="link"): + result = _l.close() + self.assertEqual(result.status, 0) + + for _b in self._brokers[0:2]: + _b.client_session.exchange_delete(exchange="fedX") + + self._teardown_brokers() + + self.verify_cleanup() + + diff --git a/qpid/cpp/src/tests/queue_flow_limit_tests.py b/qpid/cpp/src/tests/queue_flow_limit_tests.py index bdd2a21..51f9164 100644 --- a/qpid/cpp/src/tests/queue_flow_limit_tests.py +++ b/qpid/cpp/src/tests/queue_flow_limit_tests.py @@ -24,7 +24,7 @@ from qpid import datatypes, messaging from qpid.messaging import Message, Empty from threading import Thread, Lock from logging import getLogger -from time import sleep +from time import sleep, time from os import environ, popen class QueueFlowLimitTests(TestBase010): @@ -129,6 +129,27 @@ class QueueFlowLimitTests(TestBase010): self.assertEqual(i.name, "test01") self._delete_queue("test01") + # now verify that the default ratios are applied if max sizing is specified: + command = tool + \ + " --broker-addr=%s:%s " % (self.broker.host, self.broker.port) \ + + "add queue test02 --max-queue-count=10000 --max-queue-size=1000000" + cmd = popen(command) + rc = cmd.close() + self.assertEqual(rc, None) + + # now verify the settings + qs = self.qmf.getObjects(_class="queue") + for i in qs: + if i.name == "test02": + ## @todo KAG: can't get the flow size from qmf! Arrgh! + # no way to verify... + #self.assertEqual(i.arguments.get("qpid.flow_resume_count"), 55) + #self.assertEqual(i.arguments.get("qpid.flow_resume_count"), 55) + self.failIf(i.flowStopped) + break; + self.assertEqual(i.name, "test02") + self._delete_queue("test02") + def test_flow_count(self): """ Create a queue with count-based flow limit. Spawn several @@ -145,11 +166,10 @@ class QueueFlowLimitTests(TestBase010): totalMsgs = 1213 + 797 + 331 # wait until flow control is active - count = 0 + deadline = time() + 10 while self.qmf.getObjects(_objectId=oid)[0].flowStopped == False and \ - count < 10: - sleep(1); - count += 1; + time() < deadline: + pass self.failUnless(self.qmf.getObjects(_objectId=oid)[0].flowStopped) depth = self.qmf.getObjects(_objectId=oid)[0].msgDepth self.assertGreater(depth, 373) @@ -200,11 +220,10 @@ class QueueFlowLimitTests(TestBase010): totalBytes = 439 + 631 + 823 # wait until flow control is active - count = 0 + deadline = time() + 10 while self.qmf.getObjects(_objectId=oid)[0].flowStopped == False and \ - count < 10: - sleep(1); - count += 1; + time() < deadline: + pass self.failUnless(self.qmf.getObjects(_objectId=oid)[0].flowStopped) self.assertGreater(self.qmf.getObjects(_objectId=oid)[0].byteDepth, 351133) diff --git a/qpid/cpp/xml/cluster.xml b/qpid/cpp/xml/cluster.xml index 24883c5..4d83c5b 100644 --- a/qpid/cpp/xml/cluster.xml +++ b/qpid/cpp/xml/cluster.xml @@ -281,6 +281,14 @@ <field name="position" type="uint8"/> <field name="count" type="uint8"/> </control> + + <!-- Replicate a QueueObserver for a given queue. --> + <control name="queue-observer-state" code="0x39"> + <field name="queue" type="str8"/> + <field name="observer-id" type="str8"/> + <field name="state" type="map"/> <!-- "name"=value --> + </control> + </class> </amqp> diff --git a/qpid/python/qpid/messaging/driver.py b/qpid/python/qpid/messaging/driver.py index 2eb2c18..78af282 100644 --- a/qpid/python/qpid/messaging/driver.py +++ b/qpid/python/qpid/messaging/driver.py @@ -66,7 +66,7 @@ class Attachment: # XXX -DURABLE_DEFAULT=True +DURABLE_DEFAULT=False # XXX @@ -526,7 +526,7 @@ class Driver: rawlog.debug("OPEN[%s]: %s:%s", self.log_id, host, port) trans = transports.TRANSPORTS.get(self.connection.transport) if trans: - self._transport = trans(host, port) + self._transport = trans(self.connection, host, port) else: raise ConnectError("no such transport: %s" % self.connection.transport) if self._retrying and self._reconnect_log: @@ -930,6 +930,7 @@ class Engine: def resolve_declare(self, sst, lnk, dir, action): declare = lnk.options.get("create") in ("always", dir) + assrt = lnk.options.get("assert") in ("always", dir) def do_resolved(type, subtype): err = None if type is None: @@ -938,7 +939,12 @@ class Engine: else: err = NotFound(text="no such queue: %s" % lnk.name) else: - action(type, subtype) + if assrt: + expected = lnk.options.get("node", {}).get("type") + if expected and type != expected: + err = AssertionFailed(text="expected %s, got %s" % (expected, type)) + if err is None: + action(type, subtype) if err: tgt = lnk.target diff --git a/qpid/python/qpid/messaging/endpoints.py b/qpid/python/qpid/messaging/endpoints.py index 30c5850..cfc89d4 100644 --- a/qpid/python/qpid/messaging/endpoints.py +++ b/qpid/python/qpid/messaging/endpoints.py @@ -158,6 +158,7 @@ class Connection(Endpoint): self.reconnect_log = options.get("reconnect_log", True) self.address_ttl = options.get("address_ttl", 60) + self.tcp_nodelay = options.get("tcp_nodelay", False) self.options = options diff --git a/qpid/python/qpid/messaging/transports.py b/qpid/python/qpid/messaging/transports.py index 8133a45..7abaae1 100644 --- a/qpid/python/qpid/messaging/transports.py +++ b/qpid/python/qpid/messaging/transports.py @@ -17,18 +17,23 @@ # under the License. # +import socket from qpid.util import connect TRANSPORTS = {} -class tcp: +class SocketTransport: - def __init__(self, host, port): + def __init__(self, conn, host, port): self.socket = connect(host, port) + if conn.tcp_nodelay: + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) def fileno(self): return self.socket.fileno() +class tcp(SocketTransport): + def reading(self, reading): return reading @@ -52,17 +57,14 @@ try: except ImportError: pass else: - class tls: + class tls(SocketTransport): - def __init__(self, host, port): - self.socket = connect(host, port) + def __init__(self, conn, host, port): + SocketTransport.__init__(self, conn, host, port) self.tls = wrap_socket(self.socket) self.socket.setblocking(0) self.state = None - def fileno(self): - return self.socket.fileno() - def reading(self, reading): if self.state is None: return reading diff --git a/qpid/python/qpid/tests/messaging/endpoints.py b/qpid/python/qpid/tests/messaging/endpoints.py index 0977b2a..db5ec03 100644 --- a/qpid/python/qpid/tests/messaging/endpoints.py +++ b/qpid/python/qpid/tests/messaging/endpoints.py @@ -46,6 +46,10 @@ class SetupTests(Base): self.conn.open() self.ping(self.conn.session()) + def testTcpNodelay(self): + self.conn = Connection.establish(self.broker, tcp_nodelay=True) + assert self.conn._driver._transport.socket.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + def testConnectError(self): try: # Specifying port 0 yields a bad address on Windows; port 4 is unassigned @@ -111,8 +115,8 @@ class SetupTests(Base): class flaky: - def __init__(self, host, port): - self.real = real(host, port) + def __init__(self, conn, host, port): + self.real = real(conn, host, port) self.sent_count = 0 self.recv_count = 0 @@ -251,8 +255,8 @@ class ConnectionTests(Base): class hangable: - def __init__(self, host, port): - self.tcp = TRANSPORTS["tcp"](host, port) + def __init__(self, conn, host, port): + self.tcp = TRANSPORTS["tcp"](conn, host, port) self.hung = False def hang(self): @@ -1182,6 +1186,16 @@ test-link-bindings-queue; { snd.send(m) self.drain(qrcv, expected=msgs) + def testAssert1(self): + try: + snd = self.ssn.sender("amq.topic; {assert: always, node: {type: queue}}") + assert 0, "assertion failed to trigger" + except AssertionFailed, e: + pass + + def testAssert2(self): + snd = self.ssn.sender("amq.topic; {assert: always}") + NOSUCH_Q = "this-queue-should-not-exist" UNPARSEABLE_ADDR = "name/subject; {bad options" UNLEXABLE_ADDR = "\0x0\0x1\0x2\0x3"
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor