diff --git a/.gitignore b/.gitignore
index 58eae60023b9f2e2e18e81d6f2a940735a0d68a5..ba2a5fce5df810bbe49706397f5f1d473bcaadc9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,8 @@ Makefile
.version
maestro-core-*/
maestro-core-*.tar.gz
+maestro-*/
+maestro-*.tar.gz
*\#
TAGS
tags
@@ -104,3 +106,4 @@ tests/check_subscribe_local
tests/subscribe-archiver.c
tests/check_cdo_selectors
/tests/check_memlock
+MaestroConfig.cmake
diff --git a/INSTALL.md b/INSTALL.md
index 579de21e9a30d2aad1a365f219743c42f8812433..b0bee5d6746b3ea9f848c93742ba33540d75f2e9 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -1,60 +1,124 @@
-Building:
----------
+# Prerequisites
+You will need a `C compiler` capable of understanding `C11`, as well as `posix threads`.
-Prerequisites: You will need a C compiler capable of understanding C11, as well
-as posix threads.
+Also, you will need `autoreconf` and `libtool` for configuring Maestro build system.
-We are including libyaml, libcyaml and libfabric as subtrees in our tree
-and build these versions automatically with our make rules.
-By default, the OFI conductor will be built. To enable the experimental other variants
-`--enable-mpi-conductor` or `--enable-smp-conductor` at configure time (INCOMPLETE).
+deps/c-timestamp
+deps/libcyaml
+deps/libfabric
+deps/libyaml
+deps/mamba
+deps/mio
+deps/protobuf
+deps/protobuf-c
+
+
+
+#### Subtree projects and updates
+
+We are including a number of external dependency libraries as subtrees in our tree under `./deps/`
+and build these versions automatically with our make rules.
+Maestro uses the libraries it built from the subtrees and not the versions of the libraries on your system.
+
+
+| Subtree | Remote branch |
+| :------------|:----------------------------------------|
+| c-timestamp | git@github.com:chansen/c-timestamp.git |
+| libcyaml | git@github.com:tlsa/libcyaml.git master |
+| libfabric | git@github.com:ofiwg/libfabric.git v1.11.x |
+| libyaml | git@github.com:yaml/libyaml.git master |
+| mamba | git@gitlab.com:cerl/mamba.git master |
+| mio | git@gitlab.version.fz-juelich.de:10022/maestro/mio.git master|
+| protobuf | git@github.com:protocolbuffers/protobuf.git 3.10.x |
+| protobuf-c | git@github.com:protobuf-c/protobuf-c.git next |
+
+
+
+To update a dependency project (e.g. mamba) subtree please do
+
+```
+git subtree pull --prefix=deps/mamba --squash git@gitlab.com:cerl/mamba.git master
+```
+
+
+# Building
+
+## For CRAY XC systems
+Please load the following modules before configuring/building, to ensure `libfabric` will be built properly.
-For CRAY XC systems you should do
module load rdma-credentials
module load gni-headers
-before configuring/building, to ensure libfabric will be built properly.
-We are also including Version 0.1.0 of the Mamba library in deps/mamba, and a snapshot of MIO in deps/mio.
+#### jupiter system
+Do not use one of the `craype-network-ofi\*` modules at this time.
+`libfabric` builds nicely with CCE 9.0.
-To use MIO you need to add the `--with-mio` flag at configure time, and have MERO installed on the system.
+### sage prototype
+Please load the following modules before configuring/building.
+
+```
+module use $CLIENT_MOD_PATH
+module swap gnu GCC/9.3.0
+module load Autotools git binutils pkg-config libreadline
+```
+
+## Calling autoreconf
-Then
```
autoreconf -ivf
-./configure # CPPFLAGS="-I/path/to/ofi/include" LDFLAGS="-L/path/to/ofi/lib"
-make
```
-Depending on which branch you build, the default configure argumenst may include options to enable the
-gcc/llvm address sanitizer. To disable, please configure with
+## Configuring
+
```
---disable-asan
+./configure
```
-If you have the address sanitizer enabled, that may (depending on the compiler) also turn on the leak checker, and since
-we are not fully memory leak-free at this point may trigger exit-time failures. To disable it, set the environment variable
+
+or specify the install directory, e.g.
+
```
-LSAN_OPTIONS=detect_leaks=0
+./configure --prefix=$HOME/maestro
```
+#### For CRAY systems
+Be sure to use `CC=cc` on the configure line.
+
+### Configuration options
-Cray-specific (jupiter)
------------------------
-Do not use one of the craype-network-ofi\* modules at this time.
-Be sure to use CC=cc on the configure line.
-libfabric builds nicely with CCE 9.0.
+By default, the OFI conductor will be built. To enable the experimental other variants
+`--enable-mpi-pool-manager` or `--enable-smp-pool-manager` at configure time (INCOMPLETE).
+
+We are also including Version 0.1.0 of the Mamba library in deps/mamba, and a snapshot of MIO in deps/mio.
+
+To use MIO you need to add the `--with-mio` flag at configure time, and have MERO installed on the system.
-Subtree project updates
------------------------
-To update a dependency project (like mamba) subtree do
+Depending on which branch you build, the default configure arguments may include options to enable the
+gcc/llvm address sanitizer. To disable, please configure with `--disable-asan`
+
+## Build
+
+Please use
+
+```
+make
```
-git subtree pull --prefix=deps/mamba --squash git@gitlab.com:cerl/mamba.git master
+
+If you have the address sanitizer enabled, that may (depending on the compiler) also turn on the leak checker. Since
+we are not fully memory leak-free at this point, it may trigger exit-time failures. To disable it, set the environment variable
+`LSAN_OPTIONS=detect_leaks=0`
+
+## Install
+
+Please use
+
+```
+make install
```
-CI
---
+# Continuous integration
We are running gitlab-CI on the master, devel, and mvp branches.
If you notice issues you can run the gitlab CI locally using docker and the gitlab-runner tool.
diff --git a/MaestroConfig.cmake.in b/MaestroConfig.cmake.in
new file mode 100644
index 0000000000000000000000000000000000000000..7e1e7f7e507fc53b8e4dbf909692667dea279b5c
--- /dev/null
+++ b/MaestroConfig.cmake.in
@@ -0,0 +1,35 @@
+#
+# Copyright (C) 2021 HPE Computer GmbH
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+set(PREFIX @prefix@)
+
+set(Maestro_INCLUDE_DIRS ${PREFIX}/include/maestro)
+set(Maestro_LIBRARIES ${PREFIX}/lib/libmaestro.a)
diff --git a/Makefile.am b/Makefile.am
index 6362694845fdd2fe933c805726a729822e103c11..5f3bba5ee1f2c1dea634bf460ca1a597f5172617 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -33,13 +33,13 @@ ACLOCAL_AMFLAGS=-I m4
# automatic version number generation from git version
# If you want to force updating the version string you need to run
-# autoconf -f
+# autoconf -f
# in the toplevel dir. We don't do this all the time automatically to
# save development time. In a fresh git checkout the initial autoreconf does
# the right thing. Of course, building a release should thus be done in a clean
# checkout.
BUILT_SOURCES = $(top_srcdir)/.version
-EXTRA_DIST = $(top_srcdir)/.version
+EXTRA_DIST = $(top_srcdir)/.version examples
$(top_srcdir)/.version:
echo $(VERSION) > $@-t && mv $@-t $@
dist-hook:
@@ -54,7 +54,7 @@ maestro: deps
tests: maestro
lib_LTLIBRARIES = libmaestro.la
-libmaestro_la_SOURCES =
+libmaestro_la_SOURCES =
libmaestro_la_LIBADD = \
maestro/libmaestro_core.la \
protocols/libmaestro_proto.la \
@@ -65,6 +65,10 @@ libmaestro_la_LIBADD = \
# README
dist_doc_DATA = README.md
+# CMake package
+cmakepackagedir = $(libdir)/cmake/maestro/
+cmakepackage_DATA = MaestroConfig.cmake
+
if WITH_FREQUENT_TAGS
all-local: TAGS tags
endif
diff --git a/README.md b/README.md
index 78063c06c2f1e2ae97a7b82b04cd32f88ed179b4..9802421e1b0f041c8e0c7785108487831afcc086 100644
--- a/README.md
+++ b/README.md
@@ -1,28 +1,61 @@
-Maestro Core
-------------
+# Description
-This repository contains the Maestro Core Library, as developed for D3.2.
-It features the Maestro Core API, used by example code and a MVP demonstrator.
+Maestro is a data- and memory-aware middleware framework that addresses the ubiquitous problems of data movement in complex memory hierarchies that exist at multiple levels of the HPC software stack.
-Installation
-------------
+<img alt="Maestro architecture overview image" src="docs/maestro-arch-overview.png">
-Please refer to INSTALL.md
-Examples
---------
+This repository contains the Maestro Core Library, as developed for `D3.2`.
+It features the Maestro Core API, used by example codes and a MVP demonstrator.
+
+# Installation
+
+Please refer to [INSTALL.md](INSTALL.md)
+
+# Usage
+Maestro can be executed on various sizes and types of machines from a simple laptop to large HPC clusters.
+On Cray systems, please build all binaries on the service nodes (login nodes) and execute on compute nodes.
+
+## Access an installed Maestro version
+
+Please include the main `Maestro header` file in your code
+```
+#include "maestro.h"
+```
+
+Please add the `include path` and `library path` of Maestro to the compilation/linking command
+```
+-I$(MAESTRO_PATH)/include/maestro -L$(MAESTRO_PATH)/lib -lmaestro
+```
+Please `export` the path to Maestro library before running
-Please use
+```
+export LD_LIBRARY_PATH=$(MAESTRO_PATH)/lib:$LD_LIBRARY_PATH
+```
+
+where `$(MAESTRO_PATH)` is Maestro install path specified during configuration with `./configure --prefix=$(MAESTRO_PATH)`
+
+## Unit tests
+
+Build the unit tests only
+```
+make check TESTS=
+```
+
+Run the unit tests only
+```
+make check TESTS
+```
+
+Build and run the unit tests. This may take some time
```
make check
```
-to build and run the test examples.
-This may take some time.
-Limits
-------
-Maestro-core needs quite a few file descriptors and also wants to lock pages
+## Limits
+
+Maestro requires quite a few file descriptors and also locks pages
into memory for RDMA purposes. We try to give a diagnostic message if errors
are triggered that may be due to resource constraints. Still, we recommend
@@ -33,72 +66,63 @@ ulimit -l 256
to set at least 1024 file descriptors and 256k of RDMA space.
+When using a system that uses `PBSPro` or `ALPS` workload manager, please export
-Local multithreaded demo (MVP1)
--------------------------------
-
-MVP1 consists in a local multithreaded demo application. More reading (d3.2) here :
-`https://bscw.zam.kfa-juelich.de/bscw/bscw.cgi/2995531`
-
-Reference version is tagged d3.2-draft, on master branch.
-
-`make check` also builds the demo executable `demo_mvp_d3_2` in addition to examples, and runs it.
-`./run_demo.sh` permits to run the demo alone.
-
-
-Adaptive Transport demo
------------------------
+```
+export APRUN_XFER_LIMITS=1
+```
+before submiting your job to ensure enough limits for Maestro on the compute nodes.
-Pool manager interlock demo uses a three application setup, comprising one pool
-manager process, and showing GFS and MIO transport. More reading (d5.5 to appear on BSCW) and
-information on how to setup a VM to run Mero here:
-`https://gitlab.version.fz-juelich.de/maestro/maestro-mero-vm`
+## Fabric provider choice/ High-Performance interconnect usage
-Reference version is tagged d5.5-review, on master branch.
+Maestro isolates the user from the multitude of network provider choices by using
+`libfabric`, and transparently choosing 'the best' connectivity between components. Unfortunately this
+functionality is not fully working, due to issues in the upstream `libfabric` code, and in incomplete testing
+of our usage of it.
-The pool manager interlock demonstration `./tests/check_pm_interlock.sh` is
-automatically launched with make check.
+Please use
+```
+export FI_PROVIDER=provider
+```
+to force a specific fabric provider.
-fabric provider choice/ High-Performance Interconnect usage
------------------------------------------------------------
+### List of supported providers
-Maestro-core is trying hard to isolate the user from the multitude of network provider choices by using
-libfabric, and transparently choosing 'the best' connectivity between components. Unfortunately this
-functionality is not fully working, due to issues in the upstream libfabric code, and in incomplete testing
-of our usage of it.
-The safest (and lowest performance) connectivity is provided by the `sockets` provider. You can force usage of
-that by setting
+| Provider | Support |
+| :------------ |:----------------------------------|
+| sockets | TCP/IP networking |
+| verbs | Infiniband networks and slingshot |
+| gni | Cray Areis networks |
+| psm2 | Intel Omni-Path networks |
+
+
+You can execute
```
-FI_PROVIDER=sockets
+./Maestro-source-dir/deps/libfabric/util/fi_info
```
+to see all the discovered providers by libfabrics, where `Maestro-source-dir` is the directory containing Maestro source.
+
+The safest (and lowest performance) connectivity is provided by the `sockets` provider. It should work on almost any network that can support TCP/IP networking, including ethernet, IB, and GNI (Aries).
-in your environment. It should work on most any network that can support TCP/IP
-networking, including ethernet, IB, and GNI (Aries).
Usage of the `tcp` and `tcp;ofi_rxm` provider is currently broken, an upstream
issue is open.
On Cray XC systems the GNI (Aries) provider is supported. If you compile with
-the `rdma-credentials` and `gni-headers` modules loaded the GNI provider should
+the `rdma-credentials` and `gni-headers` modules loaded, the GNI provider should
be autoselected if a GNI NIC is found at runtime.
NOTE that GNI NICs on login nodes typically do not work, due to a limitation of
-the libfabric/gni driver, so you will have to run your application exclusively
+the `libfabric/gni` driver, so you will have to run your application exclusively
on compute nodes, or manually switch the components running on login nodes to
the sockets provider.
-The GNI driver can be forced by setting
-```
-FI_PROVIDER=gni
-```
-
-
-If you are using GNI you will implicitly be using Cray libdrc, a mechanism to obtain network
-authentication tokens. Maestro-core is requesting workflow-level tokens that even
+If you are using GNI you will implicitly be using Cray `libdrc`, a mechanism to obtain network
+authentication tokens. Maestro core is requesting workflow-level tokens that even
support running multiple components of a workflow from different user IDs. In
some cases the system may run out of tokens, and there is no user-level token
inquiry tool available. If you see failure of GNI startup, try running your application with
@@ -115,20 +139,86 @@ LIBDRC:CORE:DEBUG rdmacred.c:658 - finished acquire request, rc=-28
If you see this, contact your system admin to clear cached DRC credentials.
+## Examples
+
+Examples can be found in `/examples` directory. It includes currently one simple example, `single_node_pool_op.c`. It is a multi-threaded application (pthread) consists of a producer thread, and two consumer threads. `single_node_pool_op.c` is based on d3.2 of Maestro project, more reading (d3.2) [here](https://bscw.zam.kfa-juelich.de/bscw/bscw.cgi/2995531).
+
+To build the example, please use
+
+```
+make MAESTRO=$(MAESTRO_PATH)
+```
+
+Example paramters, such as `num_producers`, `num_consumers`, `num_archivers`, `cdo_size`, and `cdo_count`, can be configured using `single_node_pool_op_config.yaml` file.
+
+Before executing the binary, please `export LD_LIBRARY_PATH=$(MAESTRO_PATH)/lib:$LD_LIBRARY_PATH` and then
+
+```
+./single_node_pool_op.o
+```
+## Demos
+
+### Local multithreaded demo (MVP1)
+
+MVP1 consists in a local multithreaded demo application. More reading (d3.2) [here](https://bscw.zam.kfa-juelich.de/bscw/bscw.cgi/2995531)
+
+Reference version is tagged `d3.2-draft`, on master branch.
+
+`make check` also builds the demo executable `demo_mvp_d3_2` in addition to examples, and runs it.
+`./run_demo.sh` permits to run the demo alone.
+
+
+### Adaptive Transport demo
+
+
+Pool manager interlock demo uses a three application setup, comprising one pool
+manager process, and showing GFS and MIO transport. More reading (`d5.5` to appear on BSCW) and
+information on how to setup a VM to run Mero [here](https://gitlab.version.fz-juelich.de/maestro/maestro-mero-vm)
+
+Reference version is tagged `d5.5-review`, on master branch.
+
+The pool manager interlock demonstration [check_pm_interlock.sh](./tests/check_pm_interlock.sh) is
+automatically launched with make check.
-Documentation
--------------
-Doxygen documentation is available and compiled in `./docs` folder.
+# Documentation
-Common issues/FAQs
-------------------
+Doxygen documentation is available and compiled in [docs](./docs) folder.
+
+# Common issues/FAQs
* If you have many network interfaces/many addresses assigned to an interface
- (may happen with IPv6 rather suddenly) the libfabric setup of the pool
- manager may hit 'too many open files/errno=-24' issues. Check `ulimit -n`,
+ (may happen with `IPv6` rather suddenly) the `libfabric` setup of the pool
+ manager may hit `too many open files/errno=-24` issues. Check `ulimit -n`,
and increase the limit.
-* If you see clients stuck at JOIN time while everything else looks good, there
+* If you see clients stuck at `JOIN` time while everything else looks good, there
is a chance that your firewalling intercepts the packages.
+
+* ##### Is it safe to invalidate the pointer to data wrapped by an offered cdo, given that ownership has passed to maestro core? (We keep hold onto the cdo handle itself of course.)
+ No, the allocation that was captured in the cdo handle must not be touched until after DISPOSE. Of course you can forget the pointer you have, but you must not re-use the allocation or free it.
+
+* ##### Does a producer have to check whether an offered cdo has been consumed (DEMANDed), and wait until it is, before calling (withdraw followed by) dispose?
+ A producer cannot directly figure that out (unless you do complicated event ops). The idea is: The consumer must submit the REQUIRE before the WITHDRAW occurs. This can be accomplished by
+ 1. pre-posting the REQUIRE, or
+ 2. by posting it after observing an OFFER:before or OFFER:after event (for safety with a 'require-ack' flag or any earlier event, like DECLARE or SEAL),
+ 3. by posting it in a WITHDRAW:before with require-ack set.
+ In all these cases Maestro will ensure that the REQUIRE can be satisfied, by taking a copy (more or less eagerly, this is to be tuned], or by blocking WITHDRAW.
+
+* ##### Should withdraw be needed at all if an offered cdo is consumed by another application?
+ Every OFFER must be followed by WITHDRAW (and DISPOSE); every REQUIRE must be followed by RETRACT or DEMAND (and DISPOSE). Remember that one OFFER can satisfy many REQUIRES for the same CDO; WITHDRAW indicates that you're no longer ready to do so (and maestro needs to ensure outstanding REQUIRES can still be satisfied if their DEMAND comes in)
+
+* ##### Should dispose block until an offered cdo is consumed by another? Or will it only block if there has already been a require posted?
+ WITHDRAW may block if maestro decides that it cannot or does not want to take a copy and there is at least one outstanding REQUIRE for the CDO, or a DEMAND is still in progress. DISPOSE should never block (but may take some time -- but not related to the pool protocol).
+
+
+# Authors and acknowledgment
+
+[Data Orchestration in High Performance Computing](https://www.maestro-data.eu/) project has received funding from the European Union’s Horizon 2020 research and innovation program through grant agreement 801101.
+
+# License
+
+[BSD 3-clause License](https://gitlab.com/cerl/maestro/maestro-core/-/blob/devel/LICENSE)
+
+
diff --git a/attributes/maestro-schema.c b/attributes/maestro-schema.c
index eb7db7b9ffd700ff7178db840caba1575ab50269..ac87c0d3e79c38504f2614a78575c289afd29493 100644
--- a/attributes/maestro-schema.c
+++ b/attributes/maestro-schema.c
@@ -730,10 +730,12 @@ mstro_attribute_val__compute_size(enum mstro_stp_val_kind kind,
*val_size = sizeof(double); break;
case MSTRO_STP_STR:
case MSTRO_STP_REGEX:
- if(string)
+ if(string!=NULL)
*val_size = strlen(string)+1;
- else
+ else {
+ assert(val!=NULL);
*val_size = strlen((char*)val)+1;
+ }
break;
case MSTRO_STP_TIMESTAMP:
*val_size = sizeof(mstro_timestamp);
@@ -2516,7 +2518,8 @@ mstro_attribute_dict_set(mstro_attribute_dict dict, const char *key,
}
if(copy_value) {
- status = mstro_attribute_val__compute_size(entry->kind, NULL, val, &entry->valsize);
+ status = mstro_attribute_val__compute_size(entry->kind, NULL,
+ val, &entry->valsize);
if(status!=MSTRO_OK) {
ERR("Cannot compute value size\n");
goto BAILOUT;
@@ -2709,7 +2712,18 @@ mstro_attribute_entry_to_mapentry(const struct mstro_attribute_entry_ *entry,
case MSTRO_STP_STR:
case MSTRO_STP_REGEX:
res->val->val_case = MSTRO__POOL__AVAL__VAL_STRING;
- res->val->string = (char *)entry->val;
+ /* we need to duplicate strings, since the deallocation of
+ * protobuf messages descends into char* members, and so the
+ * entry in the dictionary where the entry is will be invalid
+ * when the message is deallocated (or vice versa) */
+ res->val->string = strdup((char *)entry->val);
+ if(res->val->string==NULL) {
+ ERR("Failed to allocat string value for KV entry\n");
+ free(res->val);
+ free(res);
+ s=MSTRO_NOMEM;
+ goto BAILOUT;
+ }
break;
case MSTRO_STP_BLOB:
res->val->val_case = MSTRO__POOL__AVAL__VAL_BYTES;
diff --git a/build-envs/README b/build-envs/README
index 234f7a826822a05acc1bf23e885adc30771bf6ba..e48c4ffd8d92db5ad508ee37fa278c4cdf285b75 100644
--- a/build-envs/README
+++ b/build-envs/README
@@ -25,6 +25,13 @@ Running things by hand
$ docker pull registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian
$ docker run --ulimit memlock=133000:133000 -i -t registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian /bin/bash
+
+ # but since you're reading this, you may be interested in debugging. In that
+ # case, to permit gdb to work inside the docker container you might want to
+ # use
+ # docker run --cap-add=SYS_PTRACE --ulimit memlock=133000:133000 -i -t registry.gitlab.com/cerl/maestro/maestro-core/buildenv/debian /bin/bash
+ # instead
+
# now you have a shell in the docker container that is currently used on gitlab for CI
$ mkdir /builds
$ cd /builds
diff --git a/build/clang/Makefile b/build/clang/Makefile
deleted file mode 100644
index 907900239586445b6d24a684d690bc33a8d3785b..0000000000000000000000000000000000000000
--- a/build/clang/Makefile
+++ /dev/null
@@ -1,51 +0,0 @@
-CC=clang
-CFLAGS_D=-O0 -g2 -Wall
-CFLAGS_R=-O2 -Wall -DNDEBUG
-LDFLAGS_D=
-LDFLAGS_R=
-
-SRCDIR=../../src
-TMPDIR_D=tmp/debug
-TMPDIR_R=tmp/release
-BINDIR_D=bin/debug
-BINDIR_R=bin/release
-
-EXAMPLES:=$(patsubst $(SRCDIR)/examples/%.peg,examples/%,$(wildcard $(SRCDIR)/examples/*.peg))
-
-BINS= \
- $(BINDIR_D)/packcc \
- $(BINDIR_R)/packcc \
- $(patsubst %,$(BINDIR_D)/%,$(EXAMPLES)) \
- $(patsubst %,$(BINDIR_R)/%,$(EXAMPLES))
-SRCS= \
- $(patsubst %,$(TMPDIR_D)/%.c,$(EXAMPLES)) \
- $(patsubst %,$(TMPDIR_D)/%.h,$(EXAMPLES)) \
- $(patsubst %,$(TMPDIR_R)/%.c,$(EXAMPLES)) \
- $(patsubst %,$(TMPDIR_R)/%.h,$(EXAMPLES))
-
-.PHONY: all clean
-
-.SECONDARY: $(SRCS)
-
-all: $(BINS)
-
-$(BINDIR_D)/packcc: $(SRCDIR)/packcc.c
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_D) -o $@ $< $(LDFLAGS_D)
-
-$(BINDIR_R)/packcc: $(SRCDIR)/packcc.c
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_R) -o $@ $< $(LDFLAGS_R)
-
-$(BINDIR_D)/examples/%: $(TMPDIR_D)/examples/%.c $(TMPDIR_D)/examples/%.h
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_D) -I. -o $@ $< $(LDFLAGS_D)
-
-$(BINDIR_R)/examples/%: $(TMPDIR_R)/examples/%.c $(TMPDIR_R)/examples/%.h
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_R) -I. -o $@ $< $(LDFLAGS_R)
-
-$(TMPDIR_D)/examples/%.c $(TMPDIR_D)/examples/%.h: $(SRCDIR)/examples/%.peg $(BINDIR_D)/packcc
- mkdir -p $(dir $@) && $(BINDIR_D)/packcc -o $(basename $@) $<
-
-$(TMPDIR_R)/examples/%.c $(TMPDIR_R)/examples/%.h: $(SRCDIR)/examples/%.peg $(BINDIR_R)/packcc
- mkdir -p $(dir $@) && $(BINDIR_R)/packcc -o $(basename $@) $<
-
-clean:
- rm -f $(BINS) $(SRCS)
diff --git a/build/gcc/Makefile b/build/gcc/Makefile
deleted file mode 100644
index eb6e4893328f3e5e5a65d33fc870d205abd73a1c..0000000000000000000000000000000000000000
--- a/build/gcc/Makefile
+++ /dev/null
@@ -1,51 +0,0 @@
-CC=gcc
-CFLAGS_D=-O0 -g2 -Wall
-CFLAGS_R=-O2 -Wall -DNDEBUG
-LDFLAGS_D=
-LDFLAGS_R=
-
-SRCDIR=../../src
-TMPDIR_D=tmp/debug
-TMPDIR_R=tmp/release
-BINDIR_D=bin/debug
-BINDIR_R=bin/release
-
-EXAMPLES:=$(patsubst $(SRCDIR)/examples/%.peg,examples/%,$(wildcard $(SRCDIR)/examples/*.peg))
-
-BINS= \
- $(BINDIR_D)/packcc \
- $(BINDIR_R)/packcc \
- $(patsubst %,$(BINDIR_D)/%,$(EXAMPLES)) \
- $(patsubst %,$(BINDIR_R)/%,$(EXAMPLES))
-SRCS= \
- $(patsubst %,$(TMPDIR_D)/%.c,$(EXAMPLES)) \
- $(patsubst %,$(TMPDIR_D)/%.h,$(EXAMPLES)) \
- $(patsubst %,$(TMPDIR_R)/%.c,$(EXAMPLES)) \
- $(patsubst %,$(TMPDIR_R)/%.h,$(EXAMPLES))
-
-.PHONY: all clean
-
-.SECONDARY: $(SRCS)
-
-all: $(BINS)
-
-$(BINDIR_D)/packcc: $(SRCDIR)/packcc.c
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_D) -o $@ $< $(LDFLAGS_D)
-
-$(BINDIR_R)/packcc: $(SRCDIR)/packcc.c
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_R) -o $@ $< $(LDFLAGS_R)
-
-$(BINDIR_D)/examples/%: $(TMPDIR_D)/examples/%.c $(TMPDIR_D)/examples/%.h
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_D) -I. -o $@ $< $(LDFLAGS_D)
-
-$(BINDIR_R)/examples/%: $(TMPDIR_R)/examples/%.c $(TMPDIR_R)/examples/%.h
- mkdir -p $(dir $@) && $(CC) $(CFLAGS_R) -I. -o $@ $< $(LDFLAGS_R)
-
-$(TMPDIR_D)/examples/%.c $(TMPDIR_D)/examples/%.h: $(SRCDIR)/examples/%.peg $(BINDIR_D)/packcc
- mkdir -p $(dir $@) && $(BINDIR_D)/packcc -o $(basename $@) $<
-
-$(TMPDIR_R)/examples/%.c $(TMPDIR_R)/examples/%.h: $(SRCDIR)/examples/%.peg $(BINDIR_R)/packcc
- mkdir -p $(dir $@) && $(BINDIR_R)/packcc -o $(basename $@) $<
-
-clean:
- rm -f $(BINS) $(SRCS)
diff --git a/build/msvc/examples/calc/calc.vcxproj b/build/msvc/examples/calc/calc.vcxproj
deleted file mode 100644
index a586bbb58eb05f56f7951f3644101f8e3f98ec6c..0000000000000000000000000000000000000000
--- a/build/msvc/examples/calc/calc.vcxproj
+++ /dev/null
@@ -1,210 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Debug|x64">
- <Configuration>Debug</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|x64">
- <Configuration>Release</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <ItemGroup>
- <CustomBuild Include="..\..\..\..\src\examples\calc.peg">
- <FileType>Document</FileType>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)tmp\$(PlatformTarget)\$(Configuration)\%(Filename).c;%(Outputs)</Outputs>
- <LinkObjects Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</LinkObjects>
- <OutputItemType Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">ClCompile</OutputItemType>
- <TreatOutputAsContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</TreatOutputAsContent>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir)tmp\$(PlatformTarget)\$(Configuration)\%(Filename).c;%(Outputs)</Outputs>
- <LinkObjects Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkObjects>
- <OutputItemType Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">ClCompile</OutputItemType>
- <TreatOutputAsContent Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</TreatOutputAsContent>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir)tmp\$(PlatformTarget)\$(Configuration)\%(Filename).c;%(Outputs)</Outputs>
- <LinkObjects Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</LinkObjects>
- <OutputItemType Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">ClCompile</OutputItemType>
- <TreatOutputAsContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</TreatOutputAsContent>
- <Outputs Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir)tmp\$(PlatformTarget)\$(Configuration)\%(Filename).c;%(Outputs)</Outputs>
- <LinkObjects Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkObjects>
- <OutputItemType Condition="'$(Configuration)|$(Platform)'=='Release|x64'">ClCompile</OutputItemType>
- <TreatOutputAsContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</TreatOutputAsContent>
- <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
- <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</DeploymentContent>
- <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
- <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</DeploymentContent>
- <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
- <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</DeploymentContent>
- <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
- <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</DeploymentContent>
- <Command Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(SolutionDir)$(PlatformTarget)\$(Configuration)\packcc -o tmp\$(PlatformTarget)\$(Configuration)\%(Filename) %(FullPath)</Command>
- <Command Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(SolutionDir)$(PlatformTarget)\$(Configuration)\packcc -o tmp\$(PlatformTarget)\$(Configuration)\%(Filename) %(FullPath)</Command>
- <Command Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(SolutionDir)$(PlatformTarget)\$(Configuration)\packcc -o tmp\$(PlatformTarget)\$(Configuration)\%(Filename) %(FullPath)</Command>
- <Command Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(SolutionDir)$(PlatformTarget)\$(Configuration)\packcc -o tmp\$(PlatformTarget)\$(Configuration)\%(Filename) %(FullPath)</Command>
- </CustomBuild>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <VCProjectVersion>16.0</VCProjectVersion>
- <ProjectGuid>{5EB5881F-0182-4E32-A3E8-6940E5016770}</ProjectGuid>
- <Keyword>Win32Proj</Keyword>
- <RootNamespace>calc</RootNamespace>
- <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
- <ProjectName>calc</ProjectName>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="Shared">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <LinkIncremental>true</LinkIncremental>
- <OutDir>$(ProjectDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <IncludePath>.\;$(IncludePath)</IncludePath>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <LinkIncremental>true</LinkIncremental>
- <OutDir>$(ProjectDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <IncludePath>.\;$(IncludePath)</IncludePath>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <LinkIncremental>false</LinkIncremental>
- <OutDir>$(ProjectDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <IncludePath>.\;$(IncludePath)</IncludePath>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <LinkIncremental>false</LinkIncremental>
- <OutDir>$(ProjectDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- <TargetName>$(ProjectName)</TargetName>
- <IncludePath>.\;$(IncludePath)</IncludePath>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- <PreBuildEvent>
- <Command>
- </Command>
- </PreBuildEvent>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file
diff --git a/build/msvc/examples/calc/calc.vcxproj.filters b/build/msvc/examples/calc/calc.vcxproj.filters
deleted file mode 100644
index 7c4e44f391f148b650560ac64f9fbfc5bb1614ae..0000000000000000000000000000000000000000
--- a/build/msvc/examples/calc/calc.vcxproj.filters
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup>
- <Filter Include="Source Files">
- <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
- <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
- </Filter>
- <Filter Include="Header Files">
- <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
- <Extensions>h;hh;hpp;hxx;hm;inl;inc;ipp;xsd</Extensions>
- </Filter>
- <Filter Include="Resource Files">
- <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
- <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
- </Filter>
- </ItemGroup>
- <ItemGroup>
- <CustomBuild Include="..\..\..\..\src\examples\calc.peg">
- <Filter>Source Files</Filter>
- </CustomBuild>
- </ItemGroup>
-</Project>
\ No newline at end of file
diff --git a/build/msvc/examples/calc/calc.vcxproj.user b/build/msvc/examples/calc/calc.vcxproj.user
deleted file mode 100644
index 0f14913f3c72094bb7b1e695e153ade04b17d5b0..0000000000000000000000000000000000000000
--- a/build/msvc/examples/calc/calc.vcxproj.user
+++ /dev/null
@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup />
-</Project>
\ No newline at end of file
diff --git a/build/msvc/msvc.sln b/build/msvc/msvc.sln
deleted file mode 100644
index 97f234da09bc5df82b98dc459f0efd972f141f90..0000000000000000000000000000000000000000
--- a/build/msvc/msvc.sln
+++ /dev/null
@@ -1,49 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.29209.62
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "packcc", "packcc.vcxproj", "{9A1A5538-7127-4B2F-9C82-3282A69F3C9C}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "calc", "examples\calc\calc.vcxproj", "{5EB5881F-0182-4E32-A3E8-6940E5016770}"
- ProjectSection(ProjectDependencies) = postProject
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C} = {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}
- EndProjectSection
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "examples", "examples", "{16C15CF5-EA71-4DEB-A758-CC52E0CA2558}"
-EndProject
-Global
- GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|x64 = Debug|x64
- Debug|x86 = Debug|x86
- Release|x64 = Release|x64
- Release|x86 = Release|x86
- EndGlobalSection
- GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Debug|x64.ActiveCfg = Debug|x64
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Debug|x64.Build.0 = Debug|x64
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Debug|x86.ActiveCfg = Debug|Win32
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Debug|x86.Build.0 = Debug|Win32
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Release|x64.ActiveCfg = Release|x64
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Release|x64.Build.0 = Release|x64
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Release|x86.ActiveCfg = Release|Win32
- {9A1A5538-7127-4B2F-9C82-3282A69F3C9C}.Release|x86.Build.0 = Release|Win32
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Debug|x64.ActiveCfg = Debug|x64
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Debug|x64.Build.0 = Debug|x64
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Debug|x86.ActiveCfg = Debug|Win32
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Debug|x86.Build.0 = Debug|Win32
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Release|x64.ActiveCfg = Release|x64
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Release|x64.Build.0 = Release|x64
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Release|x86.ActiveCfg = Release|Win32
- {5EB5881F-0182-4E32-A3E8-6940E5016770}.Release|x86.Build.0 = Release|Win32
- EndGlobalSection
- GlobalSection(SolutionProperties) = preSolution
- HideSolutionNode = FALSE
- EndGlobalSection
- GlobalSection(NestedProjects) = preSolution
- {5EB5881F-0182-4E32-A3E8-6940E5016770} = {16C15CF5-EA71-4DEB-A758-CC52E0CA2558}
- EndGlobalSection
- GlobalSection(ExtensibilityGlobals) = postSolution
- SolutionGuid = {BD6C24C6-47A2-40C5-87E8-F72C92F9D5FF}
- EndGlobalSection
-EndGlobal
diff --git a/build/msvc/packcc.vcxproj b/build/msvc/packcc.vcxproj
deleted file mode 100644
index ee0441d064d1145ba42059281a852fdbbcc73859..0000000000000000000000000000000000000000
--- a/build/msvc/packcc.vcxproj
+++ /dev/null
@@ -1,175 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Debug|x64">
- <Configuration>Debug</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|x64">
- <Configuration>Release</Configuration>
- <Platform>x64</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <VCProjectVersion>16.0</VCProjectVersion>
- <ProjectGuid>{9A1A5538-7127-4B2F-9C82-3282A69F3C9C}</ProjectGuid>
- <Keyword>Win32Proj</Keyword>
- <RootNamespace>packcc</RootNamespace>
- <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <PlatformToolset>v142</PlatformToolset>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>Unicode</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="Shared">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <LinkIncremental>true</LinkIncremental>
- <OutDir>$(SolutionDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <LinkIncremental>true</LinkIncremental>
- <OutDir>$(SolutionDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <LinkIncremental>false</LinkIncremental>
- <OutDir>$(SolutionDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <LinkIncremental>false</LinkIncremental>
- <OutDir>$(SolutionDir)$(PlatformTarget)\$(Configuration)\</OutDir>
- <IntDir>$(PlatformTarget)\$(Configuration)\</IntDir>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- <DisableLanguageExtensions>true</DisableLanguageExtensions>
- <TreatWChar_tAsBuiltInType>false</TreatWChar_tAsBuiltInType>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- <DisableLanguageExtensions>true</DisableLanguageExtensions>
- <TreatWChar_tAsBuiltInType>false</TreatWChar_tAsBuiltInType>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- <DisableLanguageExtensions>true</DisableLanguageExtensions>
- <TreatWChar_tAsBuiltInType>false</TreatWChar_tAsBuiltInType>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
- <ClCompile>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- <SDLCheck>true</SDLCheck>
- <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <ConformanceMode>true</ConformanceMode>
- <DisableLanguageExtensions>true</DisableLanguageExtensions>
- <TreatWChar_tAsBuiltInType>false</TreatWChar_tAsBuiltInType>
- </ClCompile>
- <Link>
- <SubSystem>Console</SubSystem>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\src\packcc.c" />
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file
diff --git a/build/msvc/packcc.vcxproj.filters b/build/msvc/packcc.vcxproj.filters
deleted file mode 100644
index 3de6784b22e987f2fe74ba443358ce53648d5688..0000000000000000000000000000000000000000
--- a/build/msvc/packcc.vcxproj.filters
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup>
- <Filter Include="Source Files">
- <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
- <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
- </Filter>
- <Filter Include="Header Files">
- <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
- <Extensions>h;hh;hpp;hxx;hm;inl;inc;ipp;xsd</Extensions>
- </Filter>
- <Filter Include="Resource Files">
- <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
- <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
- </Filter>
- </ItemGroup>
- <ItemGroup>
- <ClCompile Include="..\..\src\packcc.c">
- <Filter>Source Files</Filter>
- </ClCompile>
- </ItemGroup>
-</Project>
\ No newline at end of file
diff --git a/build/msvc/packcc.vcxproj.user b/build/msvc/packcc.vcxproj.user
deleted file mode 100644
index 0f14913f3c72094bb7b1e695e153ade04b17d5b0..0000000000000000000000000000000000000000
--- a/build/msvc/packcc.vcxproj.user
+++ /dev/null
@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup />
-</Project>
\ No newline at end of file
diff --git a/configure.ac b/configure.ac
index 8a029a626991cf1710feab7a83dd65da2cbffd03..5e9274d497216637c1f50b96a494a4ff08fef909 100644
--- a/configure.ac
+++ b/configure.ac
@@ -58,9 +58,18 @@ dnl silent automake requires version 1.11
m4_ifdef([AM_SILENT_RULES],
[AM_SILENT_RULES([yes])])
-AC_ARG_ENABLE([libnuma],
- AS_HELP_STRING([--disable-libnuma],
- [Disable the Linux libnuma]))
+AC_ARG_ENABLE([numa],
+ [AS_HELP_STRING([--enable-numa],[Disable libnuma usage (default: use if detected)])],
+ [enable_numa=$enableval],[enable_numa=yes])
+AS_IF([test "x$enable_numa" = xyes],
+ [AC_CHECK_HEADERS([numa.h numaif.h],
+ [AC_CHECK_LIB([numa],[numa_available],
+ AC_DEFINE([HAVE_NUMA])
+ LIBS="-lnuma $LIBS")
+ have_numa=yes],
+ [have_numa=no])])
+AM_CONDITIONAL([NUMA_ENABLED], [test "x$have_numa" = "xyes"])
+AC_SUBST([NUMA_ENABLED], [$have_numa])
AC_CONFIG_SRCDIR([maestro/version.c])
AX_CHECK_ENABLE_DEBUG()
@@ -115,7 +124,6 @@ AC_COMPILE_IFELSE(
[AC_MSG_NOTICE([C compiler understands C11 well])],
[AC_MSG_ERROR([C compiler does not handle constant initializers to structures, not fully C11 compatible])])
-
dnl check for code-coverage option. This needs to stay at the start of checks
AC_ARG_ENABLE([code-coverage],
AS_HELP_STRING([--enable-code-coverage],
@@ -167,7 +175,8 @@ AC_ARG_ENABLE([tsan],
if test "x$enable_tsan" = "xyes"; then
BUILD_TSAN=yes
AX_CHECK_COMPILE_FLAG([-fsanitize=thread -fno-omit-frame-pointer],
- [CFLAGS="${CFLAGS} -O1 -g -fsanitize=thread -fno-omit-frame-pointer"],
+ [CFLAGS="${CFLAGS} -O1 -g -fsanitize=thread -fno-omit-frame-pointer"
+ LDFLAGS="${LDFLAGS} -fsanitize=thread"],
AC_MSG_ERROR([compiler does not support -fsanitize=thread flag]))
else
BUILD_TSAN=no
@@ -415,7 +424,9 @@ AS_IF([test "x$enable_ofi_pool_manager" = "xyes"], [
AS_IF([test ! -f $srcdir/deps/libfabric/configure],
[(cd $srcdir/deps/libfabric; ./autogen.sh)])
#AX_SUBDIRS_CONFIGURE([deps/libfabric],[[--enable-embedded],[--disable-rxm],[--disable-rxd],[--disable-psm]],[],[],[])
- AX_SUBDIRS_CONFIGURE([deps/libfabric],[[--enable-embedded],[--disable-rxd],[--disable-shm],[--disable-tcp],[--enable-debug]],[],[],[])
+ dnl Configuring without kdreg to fix issue https://gitlab.com/cerl/maestro/maestro-core/-/issues/117 which relates to https://github.com/ofiwg/libfabric/issues/5313
+ dnl Whether doing so creates a performance problem or not is still to be determined
+ AX_SUBDIRS_CONFIGURE([deps/libfabric],[[--enable-embedded],[--disable-rxd],[--disable-shm],[--disable-tcp],[--with-kdreg=no],[--enable-debug]],[],[],[])
AC_MSG_NOTICE([================== done preconfiguring private libfabric library build])
])
@@ -536,6 +547,7 @@ AC_CONFIG_FILES([
deps/libyaml/src/Makefile
deps/libcyaml/Makefile
deps/c-timestamp/Makefile
+ MaestroConfig.cmake
])
dnl AC_SUBST does not preserve executable bit on scripts, so do
diff --git a/docs/maestro-arch-overview.png b/docs/maestro-arch-overview.png
new file mode 100644
index 0000000000000000000000000000000000000000..99e98dbd94ae2f85692a33ed2a198ad8c085cbc3
Binary files /dev/null and b/docs/maestro-arch-overview.png differ
diff --git a/examples/Makefile b/examples/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..5094eed71ecd2d4c4e4c740137de410bf0abb504
--- /dev/null
+++ b/examples/Makefile
@@ -0,0 +1,30 @@
+MAESTRO ?=/usr/local/
+
+CC ?= gcc
+MPICC ?= mpicc
+
+INC=-I$(MAESTRO)/include/maestro
+LDFLAGS=-L$(MAESTRO)/lib
+
+LIBS=-lmaestro
+LIBS_OMP=-fopenmp -lmaestro
+
+CFLAGS=-O3
+
+OBJs = core_bench local_pool_op
+
+
+all: local_pool_op core_bench
+
+%: %.c
+ $(CC) $(CFLAGS) $(INC) $< $(LDFLAGS) $(LIBS) -o $@
+
+%.o: %.c
+ $(MPICC) -c $(CFLAGS) $(INC) $< -o $@
+
+core_bench: omp_injector.o omp_consumer.o core_benchmark.o
+ $(MPICC) $(LDFLAGS) $^ $(LIBS_OMP) -o $@
+
+
+clean:
+ rm -f $(OBJs) *.o *CDO*
diff --git a/examples/bechmark_attributes.yaml b/examples/bechmark_attributes.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c2be382888ed3106e575fea7286edd4bf63cabfe
--- /dev/null
+++ b/examples/bechmark_attributes.yaml
@@ -0,0 +1,220 @@
+# A user-defined schema. The minimum is name and version
+schema-name: Benchmark Attributes
+schema-version: 0
+
+schema-namespace: ".maestro.benchmark."
+
+# attributes section is optional; if given it needs to have a sequence value
+maestro-attributes:
+
+# Top-level attributes
+
+ - key: "attrib_1"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_2"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_3"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_4"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_5"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_6"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_7"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_8"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_9"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_10"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_11"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_12"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_13"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_14"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_15"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_16"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_17"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_18"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_19"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_20"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_21"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_22"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_23"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_24"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_25"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_26"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_27"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_28"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_29"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_30"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_31"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_32"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_33"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_34"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "attrib_35"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
diff --git a/examples/core_benchmark.c b/examples/core_benchmark.c
new file mode 100644
index 0000000000000000000000000000000000000000..dafbf70135772023c464be41cc30c427917bf8c3
--- /dev/null
+++ b/examples/core_benchmark.c
@@ -0,0 +1,356 @@
+/* -*- mode:c -*- */
+/** @file
+ ** @brief benchmark core CDO operations, i.e., declare, offer, require, demand, withdraw, and dispose. Using multiple producers and consumers. This benchmark demonstrates also how to write an MPI+OpenMP application with Maestro.
+ **/
+
+/*
+ * Copyright (C) 2021 Hewlett Packard Enterprise
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "maestro.h"
+#include <assert.h>
+#include "maestro/logging.h"
+#include <string.h>
+#include "omp.h"
+#include "omp_injector.h"
+#include "omp_consumer.h"
+#include "mpi.h"
+
+
+#define BENCHMARK_ATTRIBUTES_YAML "./bechmark_attributes.yaml"
+#define MAX_NUM_ATTRIBUTES 35
+
+
+
+/* simplify logging */
+#define DEBUG(...) LOG_DEBUG(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define INFO(...) LOG_INFO(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define WARN(...) LOG_WARN(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define ERR(...) LOG_ERR(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+
+int main(int argc, char *argv[])
+{
+
+ int injector_id = 1; // could come from MPI_COMM_rank
+ size_t num_attributes = 0;
+ size_t size_attributes = 0;
+ size_t CDOs_per_thread = 10000; // 10k CDOs/thread by default
+ size_t nConsumers = 0; // number of consumers
+ size_t size_CDO = 0; // size of CDOs
+ mstro_status status = MSTRO_OK; // global status
+
+
+ int rank, size;
+
+ // read arguments
+ if (argc > 1)
+ {
+ num_attributes =atoi(argv[1]);
+ if (num_attributes > MAX_NUM_ATTRIBUTES)
+ {
+ printf("Maximum number of supported attributes in this benchmark is %d \n", MAX_NUM_ATTRIBUTES);
+ num_attributes = MAX_NUM_ATTRIBUTES;
+ }
+ }
+ if (argc > 2)
+ {
+ size_attributes =atoi(argv[2]);
+ }
+ if (argc > 3)
+ {
+ CDOs_per_thread =atoi(argv[3]);
+ }
+ if (argc > 4)
+ {
+ nConsumers =atoi(argv[4]);
+ }
+ if (argc > 5)
+ {
+ size_CDO =atoi(argv[5]);
+ }
+
+ /* save print time: */
+ setenv("MSTRO_LOG_LEVEL","0",1);
+
+
+ // export the path to the custom scehma
+ setenv("MSTRO_SCHEMA_LIST",BENCHMARK_ATTRIBUTES_YAML,1);
+
+ // read num of threads
+ size_t num_threads = 1;
+ // Read environment variables
+ if (getenv("OMP_NUM_THREADS") != NULL)
+ {
+ num_threads = atoi(getenv("OMP_NUM_THREADS"));
+ }
+
+ /*start MPI ... if number of ranks is 1, then start only injector component
+ that will work with local pool manager*/
+ MPI_Init(NULL,NULL);
+ MPI_Comm_rank (MPI_COMM_WORLD, &rank);
+ MPI_Comm_size (MPI_COMM_WORLD, &size);
+
+ if (size == 1)
+ {
+ // start producer
+ status = start_injector(rank, num_attributes, size_attributes, CDOs_per_thread);
+ }
+ else
+ {
+ /*If number of ranks is greater than 1,
+ 1. start a pool manager component
+ 2. broadcast PM info
+ 3.look for the num of producers and consumers and start them
+ */
+ if (rank == 0)
+ {
+ // start pool manager --- workflow, component, id
+ status = mstro_init("Tests", "Pool_Manager", 0);
+ if(status!=MSTRO_OK) {
+ ERR("Maestro Pool Manager: Failed to initialize Maestro: %d (%s)\n",
+ status, mstro_status_description(status));
+ // panic and fail
+ MPI_Abort(MPI_COMM_WORLD, -1);
+ }
+ status = mstro_pm_start();
+ if(status!=MSTRO_OK) {
+ ERR("Simple Maestro Pool Manager: Failed to start pool: %d (%s)\n",
+ status, mstro_status_description(status));
+
+ // panic and fail
+ MPI_Abort(MPI_COMM_WORLD, -1);
+ }
+
+ char *info = NULL;
+ status = mstro_pm_getinfo(&info);
+
+ if(status!=MSTRO_OK) {
+ ERR("Simple Maestro Pool Manager: Failed to obtain pool contact info: %d (%s)\n",
+ status, mstro_status_description(status));
+
+ // panic and fail
+ MPI_Abort(MPI_COMM_WORLD, -1);
+ }
+
+ assert(status == MSTRO_OK);
+
+ // get size of PM info and broadcast it
+ int info_size = strlen(info);
+ MPI_Bcast(&info_size,1,MPI_INT, 0, MPI_COMM_WORLD);
+ //broadcast the PM info
+ MPI_Bcast(info,info_size,MPI_CHAR, 0, MPI_COMM_WORLD);
+
+ //sync producers and conumers --otherwise producers may widthraw CDOs before consumed
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ }
+ else if ((rank > 0) && (rank <= nConsumers))
+ {
+ double require_time, demand_time, before, after;
+ // get PM info
+ int info_size;
+ MPI_Bcast(&info_size,1,MPI_INT, 0, MPI_COMM_WORLD);
+ //get the PM info
+ char *info = (char *) malloc(info_size*sizeof(char));
+ MPI_Bcast(info,info_size,MPI_CHAR, 0, MPI_COMM_WORLD);
+
+ // put it in the environment
+ setenv("MSTRO_POOL_MANAGER_INFO",info,1);
+
+
+ //get consumer mode
+ int consumer_mode = get_consumer_mode();
+ printf("[Consumer %d] consumer mode = %d \n",rank, consumer_mode);
+ // get the number of producers assigned to me
+ int num_producers = get_num_producers(size, nConsumers, consumer_mode);
+ printf("[Consumer %d] num_producers = %d \n",rank, num_producers );
+ // allocate producers list
+ int *producers_ids = (int *) malloc(sizeof(int)*num_producers);
+ // calculate my assigned producers
+ get_producers(rank, size, nConsumers, producers_ids, num_producers, consumer_mode);
+
+
+ for (size_t i = 0; i < num_producers; i++) {
+ printf("producer: %d \n", producers_ids[i]);
+ }
+
+ // create CDOs array
+ size_t num_CDOs = CDOs_per_thread * num_threads*num_producers; // CDOs per producer * (num_thread) * num_producers
+ printf("[Consumer %d] num_CDOs = %zu \n",rank, num_CDOs );
+
+ mstro_cdo cdos[num_CDOs];
+
+
+
+ // start consumers
+ status = mstro_init("Tests","Consumer",rank);
+ assert(MSTRO_OK == status);
+
+
+ // Require CDOs
+ before = omp_get_wtime();
+ // declare CDOs loop
+
+ status = require_CDOs(cdos, num_CDOs, producers_ids, num_producers, num_attributes, size_attributes);
+ assert(MSTRO_OK == status);
+ after = omp_get_wtime();
+ require_time = (after - before) * 1000.0*1000.0; //time in us seconds
+
+ fprintf(stdout, "Throughput (declare/require): %.5f us\n", require_time/(double) num_CDOs);
+
+
+ //sync producers and consumers -- avoid CDOs been withdrawn before require
+ MPI_Barrier(MPI_COMM_WORLD);
+
+
+ // Demand CDOs
+ before = omp_get_wtime(); // restart timer here
+ status = demand_CDOs(cdos, num_CDOs);
+ assert(MSTRO_OK == status);
+ after = omp_get_wtime();
+ demand_time = (after - before) * 1000.0*1000.0; //time in us seconds
+ fprintf(stdout, "Throughput (demand/dispose): %.5f us\n", demand_time/(double) num_CDOs);
+
+
+ // finalize Maestro
+ status = mstro_finalize();
+ assert(status == MSTRO_OK);
+
+ //clean up
+ if (producers_ids != NULL)
+ {free(producers_ids);}
+
+
+
+ }
+ else
+ {
+ double declare_time,withdraw_time, before, after;
+
+ // get PM info
+ int info_size;
+ MPI_Bcast(&info_size,1,MPI_INT, 0, MPI_COMM_WORLD);
+ //get the PM info
+ char *info = (char *) malloc(info_size*sizeof(char));
+ MPI_Bcast(info,info_size,MPI_CHAR, 0, MPI_COMM_WORLD);
+
+ // put it in the environment
+ setenv("MSTRO_POOL_MANAGER_INFO",info,1);
+
+ size_t num_CDOs = CDOs_per_thread * num_threads; // 1K per producer * (num_thread)
+
+ status = mstro_init("Tests","Injector",rank);
+
+ assert(MSTRO_OK == status);
+
+ // create CDOs array
+ mstro_cdo cdos[num_CDOs];
+
+ // create CDOs DATA
+ char *CDO_data[num_CDOs];
+ if (size_CDO != 0)
+ {
+ for (size_t i = 0; i < num_CDOs; i++) {
+ //allocate data
+ posix_memalign((void**) &CDO_data[i], (size_t) sysconf(_SC_PAGESIZE), sizeof(char)*size_CDO);
+ //fill data
+ fill_char_array(CDO_data[i], size_CDO);
+ }
+
+ }
+
+ before = omp_get_wtime();
+ // declare CDOs loop
+ status = inject_CDOs(rank, cdos, num_CDOs, num_attributes, size_attributes, size_CDO, CDO_data);
+
+ assert(MSTRO_OK == status);
+
+ after = omp_get_wtime();
+
+ declare_time = (after - before) * 1000.0*1000.0; //time in us seconds
+
+ fprintf(stdout, "#CDOs: %zu, #Threads: %zu, #Attributes: %zu, Size of attributes: %zu \n", num_CDOs, num_threads, num_attributes, size_attributes);
+ fprintf(stdout, "Throughput (declare/offer): %.5f us\n", declare_time/(double) num_CDOs);
+
+ //sync producers and consumers -- avoid CDOs been withdrawn before require
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ before = omp_get_wtime(); // restart timer here
+ // withdraw CDOs loop
+ status = withdraw_CDOs(cdos, num_CDOs);
+
+ assert(MSTRO_OK == status);
+
+ after = omp_get_wtime();
+
+ withdraw_time = (after - before) * 1000.0*1000.0; //time in us seconds
+ fprintf(stdout, "Throughput (withdraw/dispose): %.5f us\n", withdraw_time/(double) num_CDOs);
+
+ status = mstro_finalize();
+
+ assert(status == MSTRO_OK);
+
+ }
+
+
+
+ }
+
+
+ // wait for termination otherwise PM will exist before producers and consumers
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ // terminate pool manager
+ if ((rank == 0) && (size > 1))
+ {
+ status = mstro_pm_terminate();
+ if(status!=MSTRO_OK) {
+ ERR("Simple Maestro Pool Manager: Failed to shut down pool: %d (%s)\n",
+ status, mstro_status_description(status));
+
+ // panic and fail
+ MPI_Abort(MPI_COMM_WORLD, -1);
+ }
+
+ status = mstro_finalize();
+ if(status!=MSTRO_OK) {
+ ERR("Simple Maestro Pool Maestro: failed to terminate: %d (%s)\n",
+ status, mstro_status_description(status));
+
+ // panic and fail
+ MPI_Abort(MPI_COMM_WORLD, -1);
+ }
+ }
+
+ MPI_Finalize();
+
+ return 0;
+}
diff --git a/examples/local_pool_op.c b/examples/local_pool_op.c
new file mode 100644
index 0000000000000000000000000000000000000000..bb3410620fcb81c51d7fb0e90ceb38826d1ceaaa
--- /dev/null
+++ b/examples/local_pool_op.c
@@ -0,0 +1,1060 @@
+/* -*- mode:c -*- */
+/** @file
+ ** @brief Single node pool operations example
+ **/
+
+/*
+ Driver
+
+ - A multi-threaded application (pthread) consisting of
+
+ - A proxy for the workflow component (WP4) (work description)
+
+ - A producer application
+
+ - two consumer applications C1 and C2
+
+ - The producer creates CDOs (type -,1,2)
+
+ - The consumer C1 (consumer) retrieves the CDOs and performs computations on it, then drops them
+
+ - The consumer C2 (archiver) writes out the CDOs to permanent storage
+
+
+ Maestro Pool MVP
+
+ - Single node pool operation
+
+ - CDO type 0, 1 and 2 declarations
+
+ - Basic Offer-Demand functionality
+
+ - Task coupling / CDO resolution through the pool
+
+ - DEMAND operations with user-allocated buffers
+
+ - Basic interface to Mamba for type 1,2 CDOs
+ */
+
+/** Implementation:
+ * main thread:
+ * - constructs work description
+ * - posts (announces) a work description CDO for all to see.
+ * - starts threads for consumer, archiver tasks
+ * - posts CONSUMER_READY REQUIRE
+ * -- if multiple consumers, multiple such CDOs
+ * - posts ARCHIVER_READY REQUIRE
+ * -- if multiple archivers, multiple such CDOs
+ * - starts producer
+ * - waits for threads to finish
+ * - withdraws work description CDO
+ * - quits
+ *
+ * archiver:
+ * - waits for work description
+ * - posts CDO REQUIRES for all CDOs in the list
+ * - demands them one by one
+ * - writes them to disk (file name = CDO name) using Mamba tiling interface
+ * - disposes them
+ * -- when multiple archive threads run, archiver i
+ * handles CDOs with index%num_archivers==i
+ *
+ * consumer:
+ * - waits for work description
+ * - posts CDO REQUIRES for all CDOS in the list
+ * - demands them one by one
+ * - computes a checksum on the data, prints it
+ * - disposes them
+ * -- when multiple consumer threads run, consumer i
+ * handles CDOs with index%num_consumers==i
+ *
+ * producer:
+ * - waits for work description
+ * - posts CDO OFFER for all CDOs
+ * - posts CDO WITHDRAW for all CDOs
+ * - disposes them
+ * -- when multiple producer threads run, producer i
+ * handles CDOs with index%num_consumers==i
+ *
+ */
+/*
+ * Copyright (C) 2019 Cray Computer GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "maestro.h"
+#include "maestro/logging.h"
+
+#include "mamba.h"
+#include "mmb_tile_iterator.h"
+
+#include <cyaml.h>
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <assert.h>
+
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+
+
+/* simplify logging */
+#define DEBUG(...) LOG_DEBUG(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define INFO(...) LOG_INFO(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define WARN(...) LOG_WARN(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define ERR(...) LOG_ERR(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+
+
+
+/** name template of data CDOs sent around */
+#define CDO_NAME_TEMPLATE "CDO %zu"
+/** UB on length of CDO_NAME_TEMPLATE */
+#define CDO_NAME_MAX (strlen(CDO_NAME_TEMPLATE)+10)
+
+/** maximum number of CDOs in one run */
+#define CDO_COUNT_MAX 100000
+
+/** A structure holding the workflow configuration */
+struct cdo_announcement {
+ size_t num_producers; /** number of producer threads */
+ size_t num_consumers; /** number of consumer threads */
+ size_t num_archivers; /** number of archiver threads */
+ int64_t cdo_size; /** size of each CDO */
+ size_t num_entries; /** number of CDOs */
+ char *cdo_names[CDO_COUNT_MAX]; /** string names of the CDOs */
+};
+
+/** A suitable scope/layout/size string for the announcement */
+char *announcement_scope_string;
+
+/** A CDO name for the work announcement */
+#define ANNOUNCEMENT_TOKEN_NAME "CDO Announcement"
+
+/** A CDO name template for 'Consumer ready' */
+#define CONSUMER_READY_NAME_TEMPLATE "Consumer Ready %zu"
+
+/** A CDO name template for 'Archiver ready' */
+#define ARCHIVER_READY_NAME_TEMPLATE "Archiver Ready %zu"
+
+#define READY_NAME_MAX 32
+
+/** A rigid version of the workflow configuration for cyaml to fill */
+struct mvp_config_
+{
+ int num_producers;
+ int num_consumers;
+ int num_archivers;
+ int cdo_size;
+ int cdo_count;
+};
+
+typedef struct mvp_config_ * mvp_config;
+
+/** cyaml schema */
+static const cyaml_schema_field_t top_mapping_schema[] = {
+ CYAML_FIELD_INT("num_producers", CYAML_FLAG_DEFAULT | CYAML_FLAG_OPTIONAL,
+ struct mvp_config_, num_producers),
+ CYAML_FIELD_INT("num_consumers", CYAML_FLAG_DEFAULT | CYAML_FLAG_OPTIONAL,
+ struct mvp_config_, num_consumers),
+ CYAML_FIELD_INT("num_archivers", CYAML_FLAG_DEFAULT | CYAML_FLAG_OPTIONAL,
+ struct mvp_config_, num_archivers),
+ CYAML_FIELD_INT("cdo_size", CYAML_FLAG_DEFAULT | CYAML_FLAG_OPTIONAL,
+ struct mvp_config_, cdo_size),
+ CYAML_FIELD_INT("cdo_count", CYAML_FLAG_DEFAULT | CYAML_FLAG_OPTIONAL,
+ struct mvp_config_, cdo_count),
+ CYAML_FIELD_END
+};
+
+/* cyaml value schema for the top level mapping. */
+static const cyaml_schema_value_t top_schema = {
+ CYAML_VALUE_MAPPING(CYAML_FLAG_POINTER,
+ struct mvp_config_, top_mapping_schema),
+};
+
+/** Basic cyaml config */
+static const cyaml_config_t config = {
+ .log_level = CYAML_LOG_WARNING, /* Logging errors and warnings only. */
+ .log_fn = cyaml_log, /* Use the default logging function. */
+ .mem_fn = cyaml_mem, /* Use the default memory allocator. */
+};
+
+/** Call cyaml to parse config file */
+int
+mvp_config_parse(mvp_config* handle, const char* filename)
+{
+ if (handle == NULL) return MSTRO_INVARG;
+
+ cyaml_err_t err;
+ err = cyaml_load_file(filename, &config,
+ &top_schema, (cyaml_data_t **)handle, NULL);
+ if (err != CYAML_OK) {
+ fprintf(stderr, "ERROR: %s (file %s)\n", cyaml_strerror(err), filename);
+ return -1;
+ }
+
+ return 0;
+}
+
+/** announcement message receive: */
+static void
+wait_for_announcement(struct cdo_announcement *announcement)
+{
+ mstro_status s;
+ mstro_cdo announcement_cdo;
+ /* start by waiting for the announcement */
+ s = mstro_cdo_declare(ANNOUNCEMENT_TOKEN_NAME,
+ MSTRO_ATTR_DEFAULT,
+ &announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to declare announcement CDO\n");
+ abort();
+ }
+ s = mstro_cdo_attribute_set(announcement_cdo,
+ MSTRO_ATTR_CORE_CDO_RAW_PTR,
+ announcement);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to set raw-ptr attribute on announcement CDO\n");
+ abort();
+ }
+
+ s = mstro_cdo_attribute_set_yaml(announcement_cdo,
+ announcement_scope_string);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to set raw-ptr size attribute on announcement CDO\n");
+ abort();
+ }
+
+ s = mstro_cdo_require(announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to require announcement CDO\n");
+ abort();
+ }
+
+ s = mstro_cdo_demand(announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to withdraw announcement CDO\n");
+ abort();
+ }
+
+ s = mstro_cdo_dispose(announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to dispose announcement CDO\n");
+ abort();
+ }
+
+ return;
+}
+
+static void
+do_announce(struct cdo_announcement *announcement, mstro_cdo *result)
+{
+ mstro_status s;
+ mstro_cdo announcement_cdo;
+ /* start by waiting for the announcement */
+ s = mstro_cdo_declare(ANNOUNCEMENT_TOKEN_NAME,
+ MSTRO_ATTR_DEFAULT,
+ &announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to declare announcement CDO\n");
+ abort();
+ }
+ s = mstro_cdo_attribute_set(announcement_cdo,
+ MSTRO_ATTR_CORE_CDO_RAW_PTR,
+ announcement);
+
+ if(s!=MSTRO_OK) {
+ ERR("Failed to set raw-ptr attribute on announcement CDO\n");
+ abort();
+ }
+ s = mstro_cdo_attribute_set_yaml(announcement_cdo,
+ announcement_scope_string);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to set raw-ptr size attribute on announcement CDO\n");
+ abort();
+ }
+
+ s = mstro_cdo_offer(announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to offer announcement CDO\n");
+ abort();
+ }
+
+ *result = announcement_cdo;
+}
+
+static void
+withdraw_announcement(mstro_cdo announcement_cdo)
+{
+ mstro_status s;
+
+ s = mstro_cdo_withdraw(announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to withdraw announcement CDO\n");
+ abort();
+ }
+
+ s = mstro_cdo_dispose(announcement_cdo);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to dispose announcement CDO\n");
+ abort();
+ }
+
+ return;
+}
+
+/* do the equivalent of a sem_post using the given CDO name */
+static void
+cdo_sem_post(const char *name, mstro_cdo *sem)
+{
+ mstro_cdo cdo;
+ mstro_status s;
+ /* use type 0 CDO */
+ s = mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT,
+ &cdo);
+ s |= mstro_cdo_offer(cdo);
+ if(s!=MSTRO_OK) {
+ ERR("CDO sem post failed\n");
+ abort();
+ }
+ *sem = cdo;
+}
+
+static void
+cdo_sem_post_cleanup(mstro_cdo sem)
+{
+ mstro_status s = mstro_cdo_withdraw(sem);
+ s |= mstro_cdo_dispose(sem);
+
+ if(s!=MSTRO_OK) {
+ ERR("CDO sem post cleanup failed\n");
+ abort();
+ }
+}
+
+/* do the equivalent of a sem_wait using the given CDO name */
+static void
+cdo_sem_wait(const char *name)
+{
+ mstro_cdo cdo;
+ mstro_status s;
+ /* use type 0 CDO */
+ s = mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT,
+ &cdo);
+ s |= mstro_cdo_require(cdo);
+ s |= mstro_cdo_demand(cdo);
+ s |= mstro_cdo_dispose(cdo);
+
+ if(s!=MSTRO_OK) {
+ ERR("CDO sem wait failed\n");
+ abort();
+ }
+
+}
+/* Quickly flash the "Hello" token */
+void
+declare_archiver_ready(size_t index, mstro_cdo *cdo)
+{
+ char name[READY_NAME_MAX];
+ size_t s = snprintf(name, READY_NAME_MAX,
+ ARCHIVER_READY_NAME_TEMPLATE, index);
+ if(s>=READY_NAME_MAX) {
+ ERR("Failed to construct name");
+ abort();
+ }
+ cdo_sem_post(name, cdo);
+}
+
+/* Block on a "Hello" token */
+void
+wait_archiver_ready(size_t index)
+{
+ char name[READY_NAME_MAX];
+ size_t s = snprintf(name, READY_NAME_MAX,
+ ARCHIVER_READY_NAME_TEMPLATE, index);
+ if(s>=READY_NAME_MAX) {
+ ERR("Failed to construct name");
+ abort();
+ }
+ cdo_sem_wait(name);
+}
+
+void
+declare_consumer_ready(size_t index, mstro_cdo *cdo)
+{
+ char name[READY_NAME_MAX];
+ size_t s = snprintf(name, READY_NAME_MAX,
+ CONSUMER_READY_NAME_TEMPLATE, index);
+ if(s>=READY_NAME_MAX) {
+ ERR("Failed to construct name");
+ abort();
+ }
+ cdo_sem_post(name,cdo);
+}
+
+void
+wait_consumer_ready(size_t index)
+{
+ char name[READY_NAME_MAX];
+ size_t s = snprintf(name, READY_NAME_MAX,
+ CONSUMER_READY_NAME_TEMPLATE, index);
+ if(s>=READY_NAME_MAX) {
+ ERR("Failed to construct name");
+ abort();
+ }
+ cdo_sem_wait(name);
+}
+
+
+static void
+archiver_flush_to_disk(const char *name, mmbArray *a)
+{
+ size_t chunkdims = 4096;
+ mmbDimensions chunks = {1, &chunkdims};
+ mmbError stat = mmb_array_tile(a, &chunks);
+ if(stat != MMB_OK) {
+ ERR("Failed to tile mamba array (chunked tiles)\n");
+ abort();
+ }
+
+ // Loop over tiles
+ mmbTileIterator* it;
+ stat = mmb_tile_iterator_create(a, &it);
+ if(stat != MMB_OK) {
+ ERR("Failed to get tile iterator\n");
+ abort();
+ }
+
+ size_t nt;
+ stat = mmb_tile_iterator_count(it, &nt);
+ if(stat != MMB_OK) {
+ ERR("Failed to get tile iterator count\n");
+ abort();
+ }
+ stat = mmb_tile_iterator_first(it);
+ if(stat != MMB_OK) {
+ ERR("Failed to move tile iterator to first\n");
+ abort();
+ }
+ FILE *dst = fopen((const char*)name, "w");
+ if(dst==NULL) {
+ ERR("Failed to open %s for writing\n", name);
+ abort();
+ }
+ for(size_t i = 0; i < nt; i++){
+ mmbArrayTile* tile = it->tile;
+ size_t count = fwrite(&MMB_IDX_1D(tile, tile->lower[0], char),
+ 1,
+ tile->upper[0]-tile->lower[0],
+ dst);
+ if(count!= tile->upper[0]-tile->lower[0]) {
+ ERR("Incomplete write on tile %d of %s\n", i, name);
+ abort();
+ }
+ stat = mmb_tile_iterator_next(it);
+ if(stat != MMB_OK) {
+ ERR("Failed to increment tile iterator\n");
+ abort();
+ }
+ }
+ if(0!=fclose(dst)) {
+ ERR("Failed to close %s after writing\n", name);
+ abort();
+ }
+ stat = mmb_tile_iterator_destroy(it);
+ if(stat != MMB_OK) {
+ ERR("Failed to free tile iterator\n");
+ abort();
+ }
+
+ // Remove tiling
+ stat = mmb_array_untile(a);
+ if(stat != MMB_OK) {
+ ERR("Failed to untile mamba array\n");
+ abort();
+ }
+
+}
+
+/*
+ * archiver:
+ * - waits for work description
+ * - posts CDO REQUIRES for all CDOs in the list
+ * - demands them one by one
+ * - writes them to disk (file name = CDO name) using Mamba tiling interface
+ * - disposes them
+ * -- when multiple archive threads run, archiver i
+ * handles CDOs with index%num_archivers==i
+ */
+void*
+archiver_thread_fun(void *closure)
+{
+ mstro_status s;
+ size_t my_idx = * (size_t*)closure;
+ INFO("Archiver %zu starting\n", my_idx);
+
+ struct cdo_announcement *announcement
+ = malloc(sizeof(struct cdo_announcement));
+ if(announcement==NULL) {
+ ERR("Failed to allocate for incoming announcement\n");
+ abort();
+ }
+
+ wait_for_announcement(announcement);
+
+ mstro_cdo incoming[announcement->num_entries];
+ void *incoming_buffers[announcement->num_entries];
+
+ /* declare and require all that we are responsible for */
+ for(size_t i=0; i<announcement->num_entries; i++) {
+ if(i%announcement->num_archivers==my_idx) {
+ /* declare it */
+ s = mstro_cdo_declare(announcement->cdo_names[i],
+ MSTRO_ATTR_DEFAULT,
+ &(incoming[i]));
+ if(s!=MSTRO_OK) {
+ ERR("Failed to declare CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* add allocation */
+ incoming_buffers[i] = malloc(announcement->cdo_size);
+ if(incoming_buffers[i]==NULL) {
+ ERR("Cannot allocate CDO buffer for archiving\n");
+ abort();
+ }
+ s = mstro_cdo_attribute_set(incoming[i],
+ MSTRO_ATTR_CORE_CDO_RAW_PTR,
+ incoming_buffers[i]);
+ s |= mstro_cdo_attribute_set(incoming[i],
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &announcement->cdo_size);
+ // INFO("archiver cdo %d incoming buffer %p\n", i, incoming_buffers[i]);
+
+ if(s!=MSTRO_OK) {
+ ERR("Failed to add buffer to CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* require it */
+ s = mstro_cdo_require(incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to require CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ }
+ }
+ /* send ACK that we're ready */
+ INFO("Declaring Archiver %zu ready\n", my_idx);
+ mstro_cdo ready_cdo;
+ declare_archiver_ready(my_idx,&ready_cdo);
+
+ /* process all in some order */
+ for(size_t i=0; i<announcement->num_entries; i++) {
+ if(i%announcement->num_archivers==my_idx) {
+ s = mstro_cdo_demand(incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to demand CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+
+ /* extract mamba pointer */
+ mmbArray *mamba_array;
+ s = mstro_cdo_access_mamba_array(incoming[i], &mamba_array);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to extract mamba array from CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+
+ /* do write-out */
+ archiver_flush_to_disk(announcement->cdo_names[i], mamba_array);
+
+ s = mstro_cdo_dispose(incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to dispose CDO %s after archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ free(incoming_buffers[i]);
+ }
+ }
+
+ cdo_sem_post_cleanup(ready_cdo);
+ free(announcement);
+ return NULL;
+}
+
+void buf_fill_rand(const char* name, char* b, int size)
+{
+ size_t i;
+ for (i=0; i<size/sizeof(char); i++)
+ b[i] = (unsigned char)rand()%256;
+}
+
+/*
+ * producer:
+ * - waits for work description
+ * - posts CDO OFFER for all CDOs
+ * - posts CDO WITHDRAW for all CDOs
+ * - disposes them
+ * -- when multiple producer threads run, producer i
+ * handles CDOs with index%num_consumers==i
+ */
+void*
+producer_thread_fun(void *closure)
+{
+ mstro_status s;
+ size_t my_idx = * (size_t*)closure;
+ INFO("Producer %zu starting\n", my_idx);
+
+ struct cdo_announcement *announcement
+ = malloc(sizeof(struct cdo_announcement));
+ if(announcement==NULL) {
+ ERR("Failed to allocate for incoming announcement\n");
+ abort();
+ }
+
+ wait_for_announcement(announcement);
+
+ /* produce */
+ mstro_cdo outgoing[announcement->num_entries];
+ void *outgoing_buffers[announcement->num_entries];
+
+ for(size_t i=0; i<announcement->num_entries; i++) {
+ if(i%announcement->num_producers==my_idx) {
+ /* declare it */
+ s = mstro_cdo_declare(announcement->cdo_names[i],
+ MSTRO_ATTR_DEFAULT,
+ &(outgoing[i]));
+ if(s!=MSTRO_OK) {
+ ERR("Failed to declare outgoing CDO %s\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* add allocation */
+ outgoing_buffers[i] = malloc(announcement->cdo_size);
+ if(outgoing_buffers[i]==NULL) {
+ ERR("Cannot allocate outgoing CDO buffer\n");
+ abort();
+ }
+ /* add some data */
+ buf_fill_rand(announcement->cdo_names[i],
+ outgoing_buffers[i], announcement->cdo_size);
+
+ s = mstro_cdo_attribute_set(outgoing[i],
+ MSTRO_ATTR_CORE_CDO_RAW_PTR,
+ outgoing_buffers[i]);
+ s |= mstro_cdo_attribute_set(outgoing[i],
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &announcement->cdo_size);
+
+ if(s!=MSTRO_OK) {
+ ERR("Failed to add outgoing buffer to CDO %s\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* OFFER it */
+ s = mstro_cdo_offer(outgoing[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to offer CDO %s\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ }
+ }
+ /* now claim them back */
+ for(size_t i=0; i<announcement->num_entries; i++) {
+ if(i%announcement->num_producers==my_idx) {
+ s = mstro_cdo_withdraw(outgoing[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to demand outgoing CDO %s\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+
+ s = mstro_cdo_dispose(outgoing[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to dipose outgoing CDO %s\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ free(outgoing_buffers[i]);
+ }
+ }
+
+ free(announcement);
+ return NULL;
+}
+
+
+void
+mvp_checksum(const char* name, void* rawptr, uint64_t size)
+{
+ unsigned char x;
+ uint64_t i;
+
+ for (i=0,x=0;i<size; i++)
+ x ^= ((unsigned char*)rawptr)[i];
+
+ INFO("Checksum for cdo \"%s\": %d\n", name, x);
+}
+
+void
+consumer_flush_to_disk(const char *name, void *a, uint64_t size)
+{
+ char file_name [256];
+ sprintf(file_name, "consumer_%s", name);
+
+ FILE *dst = fopen((const char*)file_name, "w");
+ if(dst==NULL) {
+ ERR("Failed to open %s for writing\n", name);
+ abort();
+ }
+
+ size_t count = fwrite(a,
+ sizeof(char),
+ size/sizeof(char),
+ dst);
+ if(count != size) {
+ ERR("Incomplete write of %s (%zu of %"PRIu64")\n", file_name, count, size);
+ abort();
+ }
+ if(0!=fclose(dst)) {
+ ERR("Failed to close %s after writing\n", file_name);
+ abort();
+ }
+}
+
+/*
+ * consumer:
+ * - waits for work description
+ * - posts CDO REQUIRES for all CDOS in the list
+ * - demands them one by one
+ * - computes a checksum on the data, prints it
+ * - disposes them
+ * -- when multiple consumer threads run, consumer i
+ * handles CDOs with index%num_consumers==i
+ */
+void *
+consumer_thread_fun(void *closure)
+{
+ mstro_status s;
+ size_t my_idx = * (size_t*)closure;
+ INFO("Consumer %zu starting\n", my_idx);
+
+ struct cdo_announcement *announcement
+ = malloc(sizeof(struct cdo_announcement));
+ if(announcement==NULL) {
+ ERR("Failed to allocate for incoming announcement\n");
+ abort();
+ }
+
+ wait_for_announcement(announcement);
+
+ /* post requests */
+ mstro_cdo incoming[announcement->num_entries];
+ void *incoming_buffers[announcement->num_entries];
+
+ /* declare and require all that we are responsible for */
+ for(size_t i=0; i<announcement->num_entries; i++) {
+ if(i%announcement->num_consumers==my_idx) {
+ /* declare it */
+ s = mstro_cdo_declare(announcement->cdo_names[i],
+ MSTRO_ATTR_DEFAULT,
+ &incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to declare CDO %s for consumption\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* add allocation */
+ incoming_buffers[i] = malloc(announcement->cdo_size);
+ if(incoming_buffers[i]==NULL) {
+ ERR("Cannot allocate CDO buffer for comsumption\n");
+ abort();
+ }
+ s = mstro_cdo_attribute_set(incoming[i],
+ MSTRO_ATTR_CORE_CDO_RAW_PTR,
+ incoming_buffers[i]);
+ s |= mstro_cdo_attribute_set(incoming[i],
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &announcement->cdo_size);
+ // INFO("consumer cdo %d incoming buffer %p\n", i, incoming_buffers[i]);
+
+ if(s!=MSTRO_OK) {
+ ERR("Failed to add buffer to CDO %s for consumption\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* require it */
+ s = mstro_cdo_require(incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to require CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ }
+ }
+
+ /* send ACK that we're ready */
+ mstro_cdo ready_cdo;
+
+ INFO("Declaring Consumer %zu ready\n", my_idx);
+ declare_consumer_ready(my_idx,&ready_cdo);
+
+ /* process all in some order */
+ for(size_t i=0; i<announcement->num_entries; i++) {
+ if(i%announcement->num_consumers==my_idx) {
+ s = mstro_cdo_demand(incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to demand CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+
+ /* extract raw ptr */
+ void* rawptr;
+ enum mstro_cdo_attr_value_type type;
+ const int64_t* size;
+ s = mstro_cdo_access_ptr(incoming[i], &rawptr, NULL);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to extract raw pointer from CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ /* query the size */
+ s = mstro_cdo_attribute_get(incoming[i],
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &type, (const void**)&size);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to extract local size from CDO %s for archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+
+ /* do some work */
+ mvp_checksum(announcement->cdo_names[i], rawptr, *size);
+
+ /* write-out */
+ consumer_flush_to_disk(announcement->cdo_names[i],
+ rawptr, *size);
+
+ s = mstro_cdo_dispose(incoming[i]);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to dispose CDO %s after archiving\n",
+ announcement->cdo_names[i]);
+ abort();
+ }
+ free(incoming_buffers[i]);
+ }
+ }
+
+ /* demand data and process */
+
+
+ cdo_sem_post_cleanup(ready_cdo);
+ free(announcement);
+ return NULL;
+}
+
+
+
+#define DEFAULT_NUM_PRODUCERS 1
+#define DEFAULT_NUM_CONSUMERS 1
+#define DEFAULT_NUM_ARCHIVERS 1
+#define DEFAULT_CDO_SIZE 1024
+#define DEFAULT_CDO_COUNT 42
+
+#define CONFIG_FILE_PATH "./local_pool_op_config.yaml"
+
+/*
+ * main thread:
+ * - constructs work description
+ * - posts (announces) a work description CDO for all to see.
+ * - starts threads for consumer, archiver tasks
+ * - posts CONSUMER_READY REQUIRE
+ * -- if multiple consumers, multiple such CDOs
+ * - posts ARCHIVER_READY REQUIRE
+ * -- if multiple archivers, multiple such CDOs
+ * - starts producer
+ * - waits for threads to finish
+ * - withdraws work description CDO
+ * - quits
+*/
+int main(int argc, char **argv)
+{
+ /* verbosity */
+ putenv("MSTRO_LOG_LEVEL=1");
+ putenv("MSTRO_LOG_COLOR_ERRORS=1");
+ putenv("MMB_LOG_LEVEL=0");
+
+ mstro_status s = mstro_init("Single Node Pool Operations",
+ "Main", 0);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to initialize Maestro\n");
+ return s;
+ }
+
+ struct cdo_announcement announcement;
+ mvp_config config_handle=NULL;
+ int err;
+
+ /* constructs work description */
+ switch(argc) {
+ default:
+ WARN("Command line arguments ignored\n");
+ /* fallthrough: */
+ case 1:
+
+ err = mvp_config_parse(&config_handle, CONFIG_FILE_PATH);
+ if (err != 0) {
+ WARN("Failed to parse a config file. Using default config.\n");
+ announcement.num_producers = DEFAULT_NUM_PRODUCERS;
+ announcement.num_consumers = DEFAULT_NUM_CONSUMERS;
+ announcement.num_archivers = DEFAULT_NUM_ARCHIVERS;
+ announcement.cdo_size = DEFAULT_CDO_SIZE;
+ announcement.num_entries = DEFAULT_CDO_COUNT;
+ }
+ else {
+ announcement.num_producers = config_handle->num_producers;
+ announcement.num_consumers = config_handle->num_consumers;
+ announcement.num_archivers = config_handle->num_archivers;
+ announcement.cdo_size = config_handle->cdo_size;
+ announcement.num_entries = config_handle->cdo_count;
+ }
+
+ break;
+ }
+ assert(announcement.num_entries<=CDO_COUNT_MAX);
+
+ pthread_t producers[announcement.num_producers];
+ pthread_t consumers[announcement.num_consumers];
+ pthread_t archivers[announcement.num_archivers];
+ size_t producer_idxs[announcement.num_producers];
+ size_t consumer_idxs[announcement.num_consumers];
+ size_t archiver_idxs[announcement.num_archivers];
+
+ for(size_t i=0; i<announcement.num_entries; i++) {
+ announcement.cdo_names[i] = malloc(CDO_NAME_MAX);
+ if(announcement.cdo_names[i]==NULL) {
+ ERR("Failed to allocate CDO name\n");
+ goto BAILOUT;
+ }
+ size_t l =
+ snprintf(announcement.cdo_names[i], CDO_NAME_MAX,
+ CDO_NAME_TEMPLATE, i);
+ if(l>=CDO_NAME_MAX) {
+ ERR("CDO name does not fit:" CDO_NAME_TEMPLATE, i);
+ goto BAILOUT;
+ }
+ }
+
+ announcement_scope_string = malloc(128);
+ if(announcement_scope_string==NULL)
+ abort();
+ snprintf(announcement_scope_string, 128,
+ "scope:\n"
+ " local-size: %zu", sizeof(struct cdo_announcement));
+
+ /* posts (announces) a work description CDO for all to see */
+ mstro_cdo announcement_cdo;
+ do_announce(&announcement, &announcement_cdo);
+
+
+ /* start consumers */
+ INFO("Starting %zu consumers\n", announcement.num_consumers);
+ for(size_t i=0; i<announcement.num_consumers; i++) {
+ consumer_idxs[i] = i;
+ pthread_create(&consumers[i], NULL,
+ consumer_thread_fun,
+ &(consumer_idxs[i]));
+ }
+ /* start archivers */
+ INFO("Starting %zu archivers\n", announcement.num_archivers);
+ for(size_t i=0; i<announcement.num_archivers; i++) {
+ archiver_idxs[i] = i;
+ pthread_create(&archivers[i], NULL,
+ archiver_thread_fun,
+ &(archiver_idxs[i]));
+ }
+
+ for(size_t i=0; i<announcement.num_consumers; i++) {
+ INFO("Waiting for Consumer %zu to be ready\n");
+ wait_consumer_ready(i);
+ }
+ for(size_t i=0; i<announcement.num_archivers; i++) {
+ INFO("Waiting for Archiver %zu to be ready\n");
+ wait_archiver_ready(i);
+ }
+
+ /* start producers */
+ INFO("Starting %zu producers\n", announcement.num_producers);
+ for(size_t i=0; i<announcement.num_producers; i++) {
+ producer_idxs[i]=i;
+ pthread_create(&producers[i], NULL,
+ producer_thread_fun,
+ &(producer_idxs[i]));
+ }
+
+
+ /* wait for threads to complete */
+ for(size_t i=0; i<announcement.num_producers; i++) {
+ pthread_join(producers[i], NULL);
+ }
+ for(size_t i=0; i<announcement.num_consumers; i++) {
+ pthread_join(consumers[i], NULL);
+ }
+ for(size_t i=0; i<announcement.num_archivers; i++) {
+ pthread_join(archivers[i], NULL);
+ }
+
+ /* withdraw announcement */
+ withdraw_announcement(announcement_cdo);
+
+
+ for(size_t i=0; i<announcement.num_entries; i++) {
+ free(announcement.cdo_names[i]);
+ }
+
+ fprintf(stdout, "Sent %zu CDOs of %" PRIi64 " bytes each from %zu produce to %zu compute and %zu archiving threads\n",
+ announcement.num_entries,
+ announcement.cdo_size,
+ announcement.num_producers,
+ announcement.num_consumers,
+ announcement.num_archivers);
+
+
+
+BAILOUT:
+ if(config_handle) free(config_handle);
+
+ return s;
+}
diff --git a/examples/local_pool_op_config.yaml b/examples/local_pool_op_config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5b801519b7a027f98541937641468975c077bb06
--- /dev/null
+++ b/examples/local_pool_op_config.yaml
@@ -0,0 +1,5 @@
+num_producers: 1
+num_consumers: 1
+num_archivers: 1
+cdo_size: 1024
+cdo_count: 42
diff --git a/examples/omp_consumer.c b/examples/omp_consumer.c
new file mode 100644
index 0000000000000000000000000000000000000000..8150eaa4f047fb9cd01dd0317642985f0377a5f0
--- /dev/null
+++ b/examples/omp_consumer.c
@@ -0,0 +1,217 @@
+/* Connect to pool and subscribe to many events, archiving all CDOs */
+/*
+ * Copyright (C) 2021 Hewlett Packard Enterprise
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "omp_consumer.h"
+#include "omp_injector.h"
+
+int convert_consumer_mode(const char * consumer_mode_env)
+{
+ int result = -1;
+
+ if (strcmp(consumer_mode_env, "MSTRO_CONSUMER_SINK_ALL") == 0)
+ {
+ result = 0;
+ }
+ else if (strcmp(consumer_mode_env, "MSTRO_CONSUMER_ONE2ONE") == 0)
+ {
+ result = 1;
+ }
+ else if (strcmp(consumer_mode_env, "MSTRO_CONSUMER_ONE2TEN") == 0)
+ {
+ result = 2;
+ }
+ else if (strcmp(consumer_mode_env, "MSTRO_CONSUMER_ALL2ALL") == 0)
+ {
+ result = 3;
+ }
+ return result;
+}
+
+int get_consumer_mode()
+{
+ char * consumer_mode_env;
+ int consumer_mode = 0;
+ //get current consumer mode
+ if (getenv("MSTRO_CONSUMER_MODE") != NULL)
+ {
+ consumer_mode_env = getenv("MSTRO_CONSUMER_MODE");
+ consumer_mode = convert_consumer_mode(consumer_mode_env);
+ }
+
+ return consumer_mode;
+}
+
+int get_num_producers(int size, int nConsumers, int consumer_mode)
+{
+ int num_producers = 0;
+
+ //calculate number of producers
+ switch(consumer_mode)
+ {
+ case MSTRO_CONSUMER_SINK_ALL:
+ case MSTRO_CONSUMER_ALL2ALL:
+ //sink all data from all producer ranks, i.e. all ranks - PM and nconsumers
+ num_producers = size - 1 - nConsumers;
+ break;
+ case MSTRO_CONSUMER_ONE2ONE:
+ //sink all data from only one producer
+ num_producers = 1;
+ break;
+
+ case MSTRO_CONSUMER_ONE2TEN:
+ //sink all data from 10 producer ranks
+ num_producers = 10;
+ break;
+
+ default:
+ ERR("Incorrect MSTRO_CONSUMER_MODE \n");
+ break;
+ }
+
+ return num_producers;
+
+}
+
+
+// producer_ids and num_producers are outputs
+void get_producers(int rank, int size, int nConsumers, int *producers_ids, int num_producers, int consumer_mode)
+{
+
+
+ //calculate number of producers
+ switch(consumer_mode)
+ {
+ case MSTRO_CONSUMER_SINK_ALL:
+ case MSTRO_CONSUMER_ALL2ALL:
+ //sink all data from all producer ranks, i.e. all ranks - PM and nconsumers
+ //calculate producers
+ for(int i = 0; i< num_producers; i++)
+ {
+ producers_ids[i] = nConsumers + 1 + i;
+ }
+ break;
+ case MSTRO_CONSUMER_ONE2ONE:
+ //sink all data from only one producer
+ //asumming nProducers == nConsumers, my assignment is shifted by my rank
+ producers_ids[0] = nConsumers + rank;
+ break;
+
+ case MSTRO_CONSUMER_ONE2TEN:
+ //sink all data from 10 producer ranks
+ //calculate producers
+ for(int i = 0; i< num_producers; i++)
+ {
+ producers_ids[i] = ((rank-1) * num_producers) + 1 + i;
+ }
+ break;
+
+ default:
+ ERR("Incorrect MSTRO_CONSUMER_MODE \n");
+ break;
+ }
+
+}
+
+
+mstro_status require_CDOs(mstro_cdo *cdos, int num_CDOs, int *injector_ids, int num_injectors, size_t num_attributes, size_t size_attributes)
+{
+
+ mstro_status s = MSTRO_OK; // global status
+ int CDOs_per_inj = num_CDOs / num_injectors ;
+ size_t cdoidx, cdo_gid;
+
+
+ // declare CDOs loop
+ #pragma omp parallel for private(cdoidx, cdo_gid) reduction(| :s)
+ for(size_t i=0; i < num_injectors; i++)
+ {
+
+ DEBUG("consuming injector %d \n", injector_ids[i]);
+
+
+ for(size_t j=0; j < CDOs_per_inj; j++)
+ {
+ mstro_status s1,s2, s3;
+ cdoidx = injector_ids[i] *CDOs_per_inj + j; // CDO id = injector_id * thread_id * num_CDOs
+ cdo_gid = i*CDOs_per_inj + j;
+ char name[CDO_NAME_MAX];
+ create_name(name, cdoidx);
+
+ s3 = MSTRO_OK; // initialize its value
+ s1 = mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, & (cdos[cdo_gid]));
+
+
+ //FIXME We do not want extra attributes now --need to merge with different branch
+ // create attributes ...numbers and sizes
+ for(size_t j=0; j < num_attributes ; j++)
+ {
+ char attrib_name[ATTRIB_NAME_MAX];
+ void * data = malloc(size_attributes*sizeof(char));
+ snprintf(attrib_name, ATTRIB_NAME_MAX, ".maestro.benchmark.attrib_%lu", j+1);
+ s3 |= mstro_cdo_attribute_set(cdos[i], attrib_name, data);
+ DEBUG("%d: %s \n", s3, mstro_status_description(s3));
+ }
+
+
+ s3 |= mstro_cdo_declaration_seal(cdos[cdo_gid]);
+
+ s2= mstro_cdo_require(cdos[cdo_gid]);
+ DEBUG("[consumer] requiring %s \n", mstro_cdo_name(cdos[cdo_gid]));
+
+ s = s | s1 | s2 | s3;
+ }
+
+ }
+
+ return s;
+}
+
+
+mstro_status demand_CDOs(mstro_cdo *cdos, int num_CDOs)
+{
+
+ mstro_status s = MSTRO_OK; // global status
+
+ #pragma omp parallel for reduction(| :s)
+ for(size_t i=0; i < num_CDOs; i++)
+ {
+ mstro_status s3,s4;
+
+ s3= mstro_cdo_demand(cdos[i]);
+ DEBUG("Hey, I recieved %s \n", mstro_cdo_name(cdos[i]));
+ s4= mstro_cdo_dispose(cdos[i]);
+ s = s | s3 | s4 ;
+ }
+ return s;
+
+}
diff --git a/examples/omp_consumer.h b/examples/omp_consumer.h
new file mode 100644
index 0000000000000000000000000000000000000000..ffe6a4348bf2397622dcb819f53039c3c4a756fc
--- /dev/null
+++ b/examples/omp_consumer.h
@@ -0,0 +1,68 @@
+/* -*- mode:c -*- */
+/** @file
+ ** @brief openMP injector component
+ **/
+
+/*
+ * Copyright (C) 2021 Hewlett Packard Enterprise
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+ #ifndef mstro_consumer
+ #define mstro_consumer 1
+
+
+ #define MSTRO_CONSUMER_SINK_ALL 0
+ #define MSTRO_CONSUMER_ONE2ONE 1
+ #define MSTRO_CONSUMER_ONE2TEN 2
+ #define MSTRO_CONSUMER_ALL2ALL 3
+
+ #include "maestro.h"
+ #include "maestro/logging.h"
+ #include <string.h>
+ #include <unistd.h>
+ #include <inttypes.h>
+
+ /* simplify logging */
+ #define DEBUG(...) LOG_DEBUG(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+ #define INFO(...) LOG_INFO(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+ #define WARN(...) LOG_WARN(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+ #define ERR(...) LOG_ERR(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+
+
+
+mstro_status require_CDOs(mstro_cdo *cdos, int num_CDOs, int *injector_ids, int num_injectors, size_t num_attributes, size_t size_attributes);
+mstro_status demand_CDOs(mstro_cdo *cdos, int num_CDOs);
+void get_producers(int rank, int size, int nConsumers, int *producers_ids, int num_producers, int consumer_mode);
+int convert_consumer_mode(const char * consumer_mode_env);
+int get_num_producers(int size, int nConsumers, int consumer_mode);
+int get_consumer_mode();
+
+#endif
diff --git a/examples/omp_injector.c b/examples/omp_injector.c
new file mode 100644
index 0000000000000000000000000000000000000000..c35d2323b8596ba398f4e255412cd5152889142a
--- /dev/null
+++ b/examples/omp_injector.c
@@ -0,0 +1,179 @@
+/* -*- mode:c -*- */
+/** @file
+ ** @brief openMP injector component
+ **/
+
+/*
+ * Copyright (C) 2021 Hewlett Packard Enterprise
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+
+#include "omp_injector.h"
+
+
+
+void create_name(char *dst, size_t idx)
+{
+ snprintf(dst, CDO_NAME_MAX, "Test CDO %zu\n", idx);
+}
+
+
+mstro_status inject_CDOs(int injector_id, mstro_cdo *cdos, int num_CDOs, size_t num_attributes, size_t size_attributes, size_t size_CDO, char **CDO_data)
+{
+
+ mstro_status s = MSTRO_OK; // global status
+
+ size_t cdoidx;
+
+ // declare CDOs loop
+ #pragma omp parallel for private(cdoidx) reduction(| :s)
+ for(size_t i=0; i < num_CDOs; i++)
+ {
+ mstro_status s1,s2, s3;
+ cdoidx = injector_id *num_CDOs + i; // CDO id = injector_id * thread_id * num_CDOs
+ char name[CDO_NAME_MAX];
+ create_name(name, cdoidx);
+
+ s3 = MSTRO_OK; // initialize its value
+ s1 = mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, & (cdos[i]));
+
+
+ // create attributes ...numbers and sizes
+ for(size_t j=0; j < num_attributes ; j++)
+ {
+ char attrib_name[ATTRIB_NAME_MAX];
+ void * data = malloc(size_attributes*sizeof(char));
+ snprintf(attrib_name, ATTRIB_NAME_MAX, ".maestro.benchmark.attrib_%lu", j+1);
+ s3 |= mstro_cdo_attribute_set(cdos[i], attrib_name, data);
+ DEBUG("%d: %s \n", s3, mstro_status_description(s3));
+ }
+
+
+ // add data
+ if (size_CDO != 0)
+ {
+ s3 |= mstro_cdo_attribute_set(cdos[i],
+ MSTRO_ATTR_CORE_CDO_RAW_PTR,
+ CDO_data[i]);
+ s3 |= mstro_cdo_attribute_set(cdos[i],
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &size_CDO);
+ }
+
+
+ s3 |= mstro_cdo_declaration_seal(cdos[i]);
+
+ s2= mstro_cdo_offer(cdos[i]);
+ DEBUG("[injector] offering %s \n", mstro_cdo_name(cdos[i]));
+
+ s = s | s1 | s2 | s3;
+ }
+
+ return s;
+
+}
+
+void fill_char_array(char *array, size_t size)
+{
+ for (size_t i = 0; i < size; i++) {
+ array[i] = 'x';
+ }
+}
+
+mstro_status withdraw_CDOs(mstro_cdo *cdos, int num_CDOs)
+{
+
+ mstro_status s = MSTRO_OK; // global status
+
+ #pragma omp parallel for reduction(| :s)
+ for(size_t i=0; i < num_CDOs; i++) {
+ mstro_status s3,s4;
+
+ s3= mstro_cdo_withdraw(cdos[i]);
+ s4= mstro_cdo_dispose(cdos[i]);
+ s = s | s3 | s4 ;
+ }
+ return s;
+
+}
+
+
+mstro_status start_injector(int injector_id, size_t num_attributes, size_t size_attributes, size_t CDOs_per_thread)
+{
+
+ double declare_time,withdraw_time, before, after;
+ size_t num_threads = 1;
+
+ // Read environment variables
+ if (getenv("OMP_NUM_THREADS") != NULL)
+ {
+ num_threads = atoi(getenv("OMP_NUM_THREADS"));
+ }
+
+ mstro_status s = MSTRO_OK; // global status
+
+
+ size_t num_CDOs = CDOs_per_thread * num_threads; // 1K per producer * (num_thread)
+
+ s = mstro_init("Tests","Injector",injector_id);
+
+ assert(MSTRO_OK == s);
+
+
+
+ mstro_cdo cdos[num_CDOs];
+
+ before = omp_get_wtime();
+ // declare CDOs loop
+ s = inject_CDOs(injector_id, cdos, num_CDOs, num_attributes, size_attributes, 0, NULL);
+
+ assert(MSTRO_OK == s);
+
+ after = omp_get_wtime();
+ declare_time = (after - before) * 1000.0*1000.0; //time in us seconds
+
+ before = after; // restart timer here
+ // withdraw CDOs loop
+ s = withdraw_CDOs(cdos, num_CDOs);
+
+ assert(MSTRO_OK == s);
+
+ after = omp_get_wtime();
+
+ withdraw_time = (after - before) * 1000.0*1000.0; //time in us seconds
+ fprintf(stdout, "#CDOs: %zu, #Threads: %zu, #Attributes: %zu, Size of attributes: %zu \n", num_CDOs, num_threads, num_attributes, size_attributes);
+ fprintf(stdout, "Throughput (declare/offer): %.5f us\n", declare_time/(double) num_CDOs);
+ fprintf(stdout, "Throughput (withdraw/dispose): %.5f us\n", withdraw_time/(double) num_CDOs);
+
+ s |= mstro_finalize();
+
+ return s;
+}
diff --git a/examples/omp_injector.h b/examples/omp_injector.h
new file mode 100644
index 0000000000000000000000000000000000000000..23104f276dd88381bd00f7ab16061d7be8010d8f
--- /dev/null
+++ b/examples/omp_injector.h
@@ -0,0 +1,66 @@
+/* -*- mode:c -*- */
+/** @file
+ ** @brief openMP injector component
+ **/
+
+/*
+ * Copyright (C) 2021 Hewlett Packard Enterprise
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+ #ifndef omp_injector
+ #define omp_injector 1
+
+#include "maestro.h"
+#include <assert.h>
+#include "maestro/logging.h"
+#include <string.h>
+#include "omp.h"
+
+
+#define CDO_NAME_MAX 32
+#define ATTRIB_NAME_MAX 128
+
+/* simplify logging */
+#define DEBUG(...) LOG_DEBUG(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define INFO(...) LOG_INFO(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define WARN(...) LOG_WARN(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+#define ERR(...) LOG_ERR(MSTRO_LOG_MODULE_USER,__VA_ARGS__)
+
+
+
+void create_name(char *dst, size_t idx);
+
+mstro_status start_injector(int injector_id, size_t num_attributes, size_t size_attributes, size_t CDOs_per_thread);
+mstro_status inject_CDOs(int injector_id, mstro_cdo *cdos, int num_CDOs, size_t num_attributes, size_t size_attributes, size_t size_CDO, char **CDO_data);
+mstro_status withdraw_CDOs(mstro_cdo *cdos, int num_CDOs);
+void fill_char_array(char *array, size_t size);
+
+#endif
diff --git a/include/maestro/attributes.h b/include/maestro/attributes.h
index ecd390a637ccf41a8f81b082e0304f85cc8a6d21..5a66bafe6027b168b4c24c97da95f6e3327df601 100644
--- a/include/maestro/attributes.h
+++ b/include/maestro/attributes.h
@@ -68,6 +68,15 @@ typedef struct mstro_attribute_dict_* mstro_cdo_attributes;
Predefined symbolic key strings exist for the well-known attributes
**/
+/**@brief maestro.core.cdo.name
+ **
+ ** The name of the CDO
+ **
+ ** C-side data type: `char *'
+ ** default value: false (set automatically by mstro_cdo_declare())
+ **/
+extern const char *MSTRO_ATTR_CORE_CDO_NAME;
+
/**@brief maestro.core.cdo.allocate-now
**
** Indicate that Maestro should perform allocation before DECLARE completes
@@ -305,18 +314,20 @@ typedef struct mstro_cdo_ *mstro_cdo;
/**
** @brief Add (*key*, *val*) pair to attribute set of *cdo*
**
- ** @param[in] cdo A CDO handle
- ** @param[in] key Attribute key string
- ** @param[in] val Pointer to the value to be set
+ ** @param[in] cdo A CDO handle
+ ** @param[in] key Attribute key string
+ ** @param[in] val Pointer to the value to be set
+ ** @param[in] copy_value Create an internal allocation for the value and
+ ** copy @arg val into it
**
- ** BEWARE: The memory pointed to by val must remain valid for the
- ** entire lifetime of the CDO. Stack-allocated variables passed in by
- ** address are a source of ugly to trace bugs.
+ ** BEWARE: If copy_value is set to false, the memory pointed to by val must
+ ** remain valid for the entire lifetime of the CDO. Stack-allocated variables
+ ** passed in by address are a source of ugly to trace bugs.
**
** @returns A status code, ::MSTRO_OK on success.
**/
mstro_status
-mstro_cdo_attribute_set(mstro_cdo cdo, const char* key, void* val);
+mstro_cdo_attribute_set(mstro_cdo cdo, const char* key, void* val, bool copy_value);
/**
** @brief Retrieve value into *val_p* associated with *key* of *cdo*.
diff --git a/include/maestro/core.h b/include/maestro/core.h
index 4954c18343e471514f1471f8cf1de7bbc5ad4453..5a64354e19c7edc39c868fedbf19579596589bc8 100644
--- a/include/maestro/core.h
+++ b/include/maestro/core.h
@@ -44,12 +44,12 @@ extern "C" {
#include "maestro/status.h"
#include <stdint.h>
#include <inttypes.h>
-
+
/**@defgroup MSTRO_Core Maestro Core API
**@{
** This is the core API, as developed for D3.2
**/
-
+
/**
** @brief Initialize the maestro core
**
@@ -74,7 +74,7 @@ extern "C" {
** @param[in] workflow_name The workflow ID.
**
** @param[in] component_name The component ID.
- **
+ **
** @param[in] component_index The unique index among all processes
** using the same *component_name*.
**
@@ -84,7 +84,7 @@ extern "C" {
mstro_core_init(const char *workflow_name,
const char *component_name,
uint64_t component_index);
-
+
/**
** @brief De-initialize the maestro core
**
@@ -103,8 +103,9 @@ extern "C" {
**/
#define PRIappid PRIu64
+
/**@} (end of group MSTRO_Core) */
-
+
/* include the remaining public API */
#include "maestro/cdo.h"
#include "maestro/pool.h"
@@ -112,10 +113,9 @@ extern "C" {
#include "maestro/attributes.h"
#include "maestro/groups.h"
#include "maestro/env.h"
-
+
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* MAESTRO_CORE_H_ */
-
diff --git a/include/maestro/env.h b/include/maestro/env.h
index 0d95b73a21a19b4fbae924559b4183bcd904d61d..8129cba1f79033118804603a93a4708ec8065e14 100644
--- a/include/maestro/env.h
+++ b/include/maestro/env.h
@@ -69,7 +69,7 @@
* all modules are permitted to log. This variable can be set to
* selectively disable or enable certain modules.
*
- * The syntax is
+ * The syntax is
* component1,component2,^component3,...
*
* where 'component1' indicates that it should be included, and
@@ -96,6 +96,20 @@
*/
#define MSTRO_ENV_LOG_COLOR "MSTRO_LOG_COLOR"
+/**@brief lists the set of Schemata (in order from left to right) that should be loaded after the maestro-core schema is loaded.
+ * If set,mstro_init will merge these onto the built-ins. Default: core + ecmwf
+ * example "export MSTRO_SCHEMA_LIST=ecmwf.yaml;benchmark.yaml"
+*/
+#define MSTRO_ENV_SCHEMA_LIST "MSTRO_SCHEMA_LIST"
+
+
+/**@brief path to search for schemata (in order from left to right).
+ * Default: current directory "."
+ * example export MSTRO_SCHEMA_PATH=".:/usr/share/maestro/schemata"
+*/
+#define MSTRO_ENV_SCHEMA_PATH "MSTRO_SCHEMA_PATH"
+
+
/**@brief enable coloring of error log messages
*
* If set, enable coloring of error and warning messages. May not be a good idea if logging to syslog (see @ref MSTRO_LOG_DST).
@@ -191,12 +205,23 @@
#define MSTRO_ENV_TRANSPORT_DEFAULT "MSTRO_TRANSPORT_DEFAULT"
+/**
+ ** @brief Directory for GFS transport
+ **
+ ** Must be visible on all workflow components that want to use GFS
+ ** transport. Should be a high-performance global file system location.
+ **
+ ** Default is "./", the directory where the application is started
+ ** (which will often work, but not always).
+ **/
+#define MSTRO_ENV_TRANSPORT_GFS_DIR "MSTRO_TRANSPORT_GFS_DIR"
+
/**
** @brief Path to the MIO transport config file
**
- ** This is needed to initialize MIO
+ ** This is needed to initialize MIO
**
- **/
+ **/
#define MSTRO_ENV_MIO_CONFIG "MSTRO_MIO_CONFIG"
/**@} (end of group MSTRO_ENV) */
diff --git a/include/maestro/i_cdo.h b/include/maestro/i_cdo.h
index d4e2fbd4203dee4eda01434746b401bca5126270..8200877e463b4c04c228329eded3f8d793f8949e 100644
--- a/include/maestro/i_cdo.h
+++ b/include/maestro/i_cdo.h
@@ -156,7 +156,14 @@ typedef uint32_t mstro_cdo_state;
/** The NIL UUID */
#define MSTRO_CDO_ID_INVALID { .id = {0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 } }
-/** return a string describing the CDO state @arg s */
+/** return a string describing the CDO state @arg s
+ **
+ ** The string is owned by this function, but this function can safely
+ ** be called from multiple threads (the buffer is thread-local). The
+ ** buffer will be re-used by subsequent calls in the same thread
+ ** after a small number of invocations (currently: 3, see @ref
+ ** MAX_STATE_DESCRIPTION_BUFFERS).
+ **/
const char *
mstro_cdo_state_describe(mstro_cdo_state s);
@@ -321,6 +328,10 @@ struct mstro_cdo_ {
char *name; /**< user-provided name */
+ int preferred_numa_node; /**< the preferred NUMA node for this CDO. -1
+ for don't care. Typically set from raw-ptr
+ or mamba-array information */
+
/* Cached copies of the data attributes. Caching occirs at SEAL
* time, after which the attribute setters will refuse chaning the
* dictionary entries, so caching is safe */
diff --git a/include/maestro/i_globals.h b/include/maestro/i_globals.h
index 60c432eb04abf1e235611fad00ba8ad2776796d0..75fab2a90c9821cb72f43e5a8601fc4584a1845d 100644
--- a/include/maestro/i_globals.h
+++ b/include/maestro/i_globals.h
@@ -95,7 +95,7 @@ extern mmbMemSpace *g_default_cdo_memspace;
/** the default (DRAM) memory interface */
extern mmbMemInterface *g_default_cdo_interface;
-
+
/** MIO can be compiled in, but run time config may be missing, or MIO may fail
* to initialize. This flag is TRUE if MIO is properly initialized and usable
* */
@@ -105,12 +105,14 @@ extern bool g_mio_available;
/** the app token used in communicating to the pool manager. Set after successfully JOINing, cleared after LEAVEing */
extern Mstro__Pool__Apptoken g_pool_apptoken;
-/** the app ID (packed version of @ref g_pool_app_id) used in communicating with the pool manager. */
+/** the app ID (packed version of @ref g_pool_app_id) used in communicating with the pool manager. */
extern Mstro__Pool__Appid g_pool_appid;
/** the fundamental built-in attribute schema. Filled early in mstro_core_init(), then constant */
extern mstro_schema g_mstro_core_schema_instance;
+/** flag set if numa is working */
+extern bool g_have_numa;
/** flag set if we are connected to a pool manager service */
extern bool g_have_pool_manager;
@@ -143,6 +145,8 @@ union mstro_component_descriptor {
char workflow_name[MSTRO_WORKFLOW_NAME_MAX];
/** the component name */
char component_name[MSTRO_WORKFLOW_NAME_MAX];
+ /** the list of user schemas */
+ char schema_list[MSTRO_WORKFLOW_NAME_MAX];
/** the maestro core version */
char version[128];
}; /*<** descriptor data */
@@ -159,4 +163,14 @@ extern union mstro_component_descriptor g_component_descriptor;
| (MSTRO_POOL_PROTOCOL_VERSION_MINOR << 8) \
| (MSTRO_POOL_PROTOCOL_VERSION_PATCH << 0)))
+/** separators for user defined schema lists and paths enviroment variables */
+#define SCHEMA_LIST_SEP ";"
+#define SCHEMA_PATH_SEP ":"
+
+/** the precomputed default path for GFS transport (incl. trailing '/') */
+extern char *g_mstro_transport_gfs_dir;
+/** string length of @ref g_mstro_transport_gfs_dir */
+extern size_t g_mstro_transport_gfs_dir_len;
+
+
#endif /* MAESTRO_I_GLOBALS_H_ */
diff --git a/maestro/Makefile.am b/maestro/Makefile.am
index 26b9feedcccab128d44ca5b7f9347932cd1197dd..e85e2723928c4a64a01f6db4e5aec1f2bef776ff 100644
--- a/maestro/Makefile.am
+++ b/maestro/Makefile.am
@@ -68,6 +68,7 @@ libmaestro_core_la_SOURCES = \
i_event.h event.c \
cdo_sel_parse.c cdo_sel_parse.h \
i_groups.h groups.c \
+ i_maestro_numa.h \
memlock.c \
cdo-attributes-default.txt
@@ -78,6 +79,10 @@ cdo_sel_parse.c cdo_sel_parse.h: cdo_sel_parse.peg
libmaestro_core_la_LIBADD=$(top_builddir)/deps/libcyaml/libcyaml.la $(top_builddir)/deps/mamba/libmmb.la $(top_builddir)/deps/c-timestamp/libtimestamp.la
+if NUMA_ENABLED
+libmaestro_core_la_LIBADD+=-lnuma
+endif
+
# Conductor implementation varies depending on configuration
if WITH_OFI_POOL_MANAGER
libmaestro_core_la_SOURCES+=pool_manager_ofi.c ofi.c drc.c
diff --git a/maestro/attributes.c b/maestro/attributes.c
index e9105e845c3564fbe1abd80ec21a919ae77a4b7b..501170a41d27247172d53cbff399ab8567562a55 100644
--- a/maestro/attributes.c
+++ b/maestro/attributes.c
@@ -50,6 +50,9 @@
/* FIXME: we should have a table of name/type/... and make the const
* char* refer to the entries in that */
+const char *MSTRO_ATTR_CORE_CDO_NAME =
+ ".maestro.core.cdo.name";
+
const char *MSTRO_ATTR_CORE_CDO_ALLOCATE_NOW =
".maestro.core.cdo.allocate-now";
@@ -124,7 +127,7 @@ mstro_cdo_attr_table__destroy(mstro_cdo_attr_table tab)
}
mstro_status
-mstro_cdo_attribute_set(mstro_cdo cdo, const char* key, void* val)
+mstro_cdo_attribute_set(mstro_cdo cdo, const char* key, void* val, bool copy_value)
{
if(cdo==NULL || key==NULL)
return MSTRO_INVARG;
@@ -160,7 +163,7 @@ mstro_cdo_attribute_set(mstro_cdo cdo, const char* key, void* val)
mstro_status status
= mstro_attribute_dict_set(cdo->attributes, fqkey,
MSTRO_CDO_ATTR_VALUE_INVALID, /* we dont help in checking */
- val, false);
+ val, copy_value);
if(tmpfqkey)
free(tmpfqkey);
return status;
diff --git a/maestro/cdo.c b/maestro/cdo.c
index 37f25ccaacc2b9e92ac52866c76956aded77c3ec..4a340c8a7349e6658470ccf3194d4479ca9435a8 100644
--- a/maestro/cdo.c
+++ b/maestro/cdo.c
@@ -46,6 +46,7 @@
#include "maestro/status.h"
#include "maestro/i_statistics.h"
+#include "i_maestro_numa.h"
#include "transformation/transformation.h"
#include <mamba.h>
@@ -144,6 +145,9 @@ mstro_cdo__alloc(void)
}
/* we need to zero out the hash-table relevant parts. For simplicity we zero everything */
memset(res, 0, sizeof(struct mstro_cdo_));
+
+ /* default: same region as handle is in */
+ res->preferred_numa_node = mstro_numa__node_for_addr(res);
/* initialize parts that need special handling */
atomic_init(&(res->state), MSTRO_CDO_STATE_INVALID);
@@ -448,10 +452,10 @@ mstro_cdo__create_mamba_array(mstro_cdo cdo)
int64_t* ndims, *patt, *localsz, *elsz;
size_t* dimsz;
- DEBUG("Sync'ing mamba_array and attributes (CDO `%s`)\n", cdo->name);
-
s = mstro_cdo_attribute_get(cdo, MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE, NULL, (const void**)&localsz);
+ DEBUG("Sync'ing mamba_array (size: %zu) and attributes (CDO `%s`)\n", (size_t)*localsz, cdo->name);
+
/* fetching minimal set of attributes for custom layout */
s |= mstro_cdo_attribute_get(cdo, MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE,
NULL, (const void**)&elsz);
@@ -524,6 +528,7 @@ mstro_cdo__create_mamba_array(mstro_cdo cdo)
}
if(me != MMB_OK) {
+ status = MSTRO_FAIL;
ERR("Failed to create mamba array\n");
goto BAILOUT;
}
@@ -639,6 +644,47 @@ mstro_cdo_declaration_seal(mstro_cdo cdo)
goto BAILOUT;
}
}
+
+ /* Ensure a sane name is filled in. The attribute is, in fact, only
+ * needed for subscription handling (so users can match on names the
+ * same way that they can for other attributes).
+ *
+ * The user does not typically set the name attribute explicitly,
+ * but if they do we need to ensure it's not in disagreement with
+ * the name specified at mstro_cdo_declare() time. */
+ const void *cdo_name_attrval=NULL;
+ enum mstro_cdo_attr_value_type valtype;
+
+ mstro_status s_cdoname = mstro_attribute_dict_get(
+ cdo->attributes, MSTRO_ATTR_CORE_CDO_NAME,
+ &valtype, &cdo_name_attrval, NULL, false);
+ if(s_cdoname==MSTRO_OK || s_cdoname==MSTRO_NOENT) {
+ const char *n_cdo = mstro_cdo_name(cdo);
+ if(s_cdoname==MSTRO_OK) {
+ /* ok, already have a name, check it matches */
+ const char *n_attr = (const char *)cdo_name_attrval;
+
+ if(0!=strcmp(n_cdo,n_attr)) {
+ ERR("CDO |%s| has name attribute set to |%s|, overriding\n",
+ cdo->name, cdo_name_attrval);
+ }
+ }
+
+ s_cdoname = mstro_attribute_dict_set(cdo->attributes,
+ MSTRO_ATTR_CORE_CDO_NAME,
+ MSTRO_CDO_ATTR_VALUE_cstring,
+ &n_cdo, true);
+ if(s_cdoname!=MSTRO_OK) {
+ ERR("Failed to set CDO name attribute: %d (%s)\n", s_cdoname,
+ mstro_status_description(s_cdoname));
+ status = s_cdoname;
+ goto BAILOUT;
+ }
+ } else {
+ ERR("Failed to inquire about CDO name attribute\n");
+ status = s_cdoname;
+ goto BAILOUT;
+ }
/* we're done, but we need to ensure we've seen the DECLARE_ACK with
* the global ID by now */
@@ -701,8 +747,8 @@ mstro_cdo_declaration_seal(mstro_cdo cdo)
idstr, &cdo->id,
DEBUG("CDO %s (id %s) now sealed\n",
cdo->name, idstr););
-
- status = MSTRO_OK;
+
+ status = MSTRO_OK;
BAILOUT:
return status;
@@ -1592,29 +1638,43 @@ struct {
/** a reasonably safe way to count the number of entries in an array of structures */
#define COUNT_OF(x) ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x])))))
+
+/** we support this many concurrent state buffers in flight per
+ * thread. Hideous, but callers typically will use this function in a
+ * PRINT statement, with up to two states (to compare them) at most
+ */
+
+#define MAX_STATE_DESCRIPTION_BUFFERS 3
const char *
mstro_cdo_state_describe(mstro_cdo_state s)
{
- static _Thread_local char buf[sizeof(states_and_names)];
+ static _Thread_local char buf[MAX_STATE_DESCRIPTION_BUFFERS][sizeof(states_and_names)];
+ static _Thread_local size_t bufid = 0;
- buf[0]='\0';
+ // switch to new buffer
+ bufid = (bufid+1) % MAX_STATE_DESCRIPTION_BUFFERS;
+
+ buf[bufid][0]='\0';
if(s==MSTRO_CDO_STATE_INVALID)
return "INVALID";
else {
for(size_t i=0; i<COUNT_OF(states_and_names); i++) {
if(s&states_and_names[i].state) {
- strcat(buf, states_and_names[i].name);
+ strcat(buf[bufid], states_and_names[i].name);
}
}
- return buf+1; /* cut off leading '|' we put there for simplicity
+
+ return buf[bufid]+1; /* cut off leading '|' we put there for simplicity
* of the loop above */
}
}
mstro_status
-mstro_cdo_attributes_update_incoming(mstro_cdo cdo, const Mstro__Pool__Attributes *attr,
- size_t num_precious_attr, const char **precious_attributes)
+mstro_cdo_attributes_update_incoming(mstro_cdo cdo,
+ const Mstro__Pool__Attributes *attr,
+ size_t num_precious_attr,
+ const char **precious_attributes)
{
if(cdo==NULL || attr==NULL)
return MSTRO_INVARG;
@@ -1791,7 +1851,7 @@ mstro_cdo_allocate_data(mstro_cdo cdo)
MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE,
MSTRO_CDO_ATTR_VALUE_pointer, dimsz, true);
if (s != MSTRO_OK) {
- ERR("Can't sync mamba array because attribute_set failed\n");
+ ERR("Can't sync mamba array because dict_set failed\n");
return MSTRO_FAIL;
}
} else {
@@ -1815,6 +1875,7 @@ mstro_cdo_allocate_data(mstro_cdo cdo)
}
/* now we know a size */
int64_t size = *(const int64_t*)valp;
+ DEBUG("size (%" PRIi64 ")\n", size);
if(size<0 && cdo->raw_ptr!=NULL) {
ERR("CDO |%s| has negative local-size but non-NULL raw-ptr, unsupported\n",
@@ -1827,6 +1888,8 @@ mstro_cdo_allocate_data(mstro_cdo cdo)
ERR("Failed to create mmbArray wrapper for raw-ptr\n");
return s;
}
+ DEBUG("size (%" PRIi64 ")\n", size);
+
}
return MSTRO_OK;
diff --git a/maestro/cdo_sel_parse.c b/maestro/cdo_sel_parse.c
index a8612ff82ebce2272e71cd7ef1ecf188618e4fdb..c6f1aa9215996e53817aa83f3e60c16cfca2cb2b 100644
--- a/maestro/cdo_sel_parse.c
+++ b/maestro/cdo_sel_parse.c
@@ -1429,7 +1429,23 @@ static void pcc_action_value_1(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__pc
static void pcc_action_value_2(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__pcc_in, pcc_value_t *__pcc_out) {
#define auxil (__pcc_ctx->auxil)
#define __ (*__pcc_out)
-#define i (*__pcc_in->data.leaf.values.buf[2])
+#define n (*__pcc_in->data.leaf.values.buf[2])
+#define _0 pcc_get_capture_string(__pcc_ctx, &__pcc_in->data.leaf.capt0)
+#define _0s ((const)__pcc_in->data.leaf.capt0.range.start)
+#define _0e ((const)__pcc_in->data.leaf.capt0.range.end)
+ __=n; DEBUG("numeric value\n");
+#undef _0e
+#undef _0s
+#undef _0
+#undef n
+#undef __
+#undef auxil
+}
+
+static void pcc_action_value_3(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__pcc_in, pcc_value_t *__pcc_out) {
+#define auxil (__pcc_ctx->auxil)
+#define __ (*__pcc_out)
+#define i (*__pcc_in->data.leaf.values.buf[3])
#define _0 pcc_get_capture_string(__pcc_ctx, &__pcc_in->data.leaf.capt0)
#define _0s ((const)__pcc_in->data.leaf.capt0.range.start)
#define _0e ((const)__pcc_in->data.leaf.capt0.range.end)
@@ -1593,7 +1609,7 @@ static void pcc_action_string_0(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__p
__=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
__->strval = strdup(_1);
if(__->strval==NULL) {
- ERR("Failed to allocate string value (for regex) parser node\n");
+ ERR("Failed to allocate string value (for string) parser node\n");
abort();
}
#undef _1e
@@ -1618,7 +1634,7 @@ static void pcc_action_string_1(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__p
__=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
__->strval = strdup(_2);
if(__->strval==NULL) {
- ERR("Failed to allocate string value (for regex) parser node\n");
+ ERR("Failed to allocate string value (for string) parser node\n");
abort();
}
#undef _2e
@@ -1631,6 +1647,58 @@ static void pcc_action_string_1(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__p
#undef auxil
}
+static void pcc_action_numeric_val_0(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__pcc_in, pcc_value_t *__pcc_out) {
+#define auxil (__pcc_ctx->auxil)
+#define __ (*__pcc_out)
+#define _0 pcc_get_capture_string(__pcc_ctx, &__pcc_in->data.leaf.capt0)
+#define _0s ((const)__pcc_in->data.leaf.capt0.range.start)
+#define _0e ((const)__pcc_in->data.leaf.capt0.range.end)
+#define _1 pcc_get_capture_string(__pcc_ctx, __pcc_in->data.leaf.capts.buf[0])
+#define _1s __pcc_in->data.leaf.capts.buf[0]->range.start
+#define _1e __pcc_in->data.leaf.capts.buf[0]->range.end
+ // signed integer, stored as string
+ __=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
+ __->strval = strdup(_1);
+ if(__->strval==NULL) {
+ ERR("Failed to allocate string value (for integral) parser node\n");
+ abort();
+ }
+#undef _1e
+#undef _1s
+#undef _1
+#undef _0e
+#undef _0s
+#undef _0
+#undef __
+#undef auxil
+}
+
+static void pcc_action_numeric_val_1(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__pcc_in, pcc_value_t *__pcc_out) {
+#define auxil (__pcc_ctx->auxil)
+#define __ (*__pcc_out)
+#define _0 pcc_get_capture_string(__pcc_ctx, &__pcc_in->data.leaf.capt0)
+#define _0s ((const)__pcc_in->data.leaf.capt0.range.start)
+#define _0e ((const)__pcc_in->data.leaf.capt0.range.end)
+#define _2 pcc_get_capture_string(__pcc_ctx, __pcc_in->data.leaf.capts.buf[1])
+#define _2s __pcc_in->data.leaf.capts.buf[1]->range.start
+#define _2e __pcc_in->data.leaf.capts.buf[1]->range.end
+ // exponential notation
+ __=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
+ __->strval = strdup(_2);
+ if(__->strval==NULL) {
+ ERR("Failed to allocate string value (for exponential numeric) parser node\n");
+ abort();
+ }
+#undef _2e
+#undef _2s
+#undef _2
+#undef _0e
+#undef _0s
+#undef _0
+#undef __
+#undef auxil
+}
+
static void pcc_action_space_0(mstro_csq_context_t *__pcc_ctx, pcc_thunk_t *__pcc_in, pcc_value_t *__pcc_out) {
#define auxil (__pcc_ctx->auxil)
#define __ (*__pcc_out)
@@ -1658,6 +1726,7 @@ static pcc_thunk_chunk_t *pcc_evaluate_rule_regex_i(mstro_csq_context_t *ctx);
static pcc_thunk_chunk_t *pcc_evaluate_rule_identifier(mstro_csq_context_t *ctx);
static pcc_thunk_chunk_t *pcc_evaluate_rule_simple_regex(mstro_csq_context_t *ctx);
static pcc_thunk_chunk_t *pcc_evaluate_rule_string(mstro_csq_context_t *ctx);
+static pcc_thunk_chunk_t *pcc_evaluate_rule_numeric_val(mstro_csq_context_t *ctx);
static pcc_thunk_chunk_t *pcc_evaluate_rule_space(mstro_csq_context_t *ctx);
static pcc_thunk_chunk_t *pcc_evaluate_rule_EOF(mstro_csq_context_t *ctx);
@@ -2488,7 +2557,7 @@ L0000:;
static pcc_thunk_chunk_t *pcc_evaluate_rule_value(mstro_csq_context_t *ctx) {
pcc_thunk_chunk_t *const chunk = pcc_thunk_chunk__create(ctx->auxil);
chunk->pos = ctx->pos;
- pcc_value_table__resize(ctx->auxil, &chunk->values, 3);
+ pcc_value_table__resize(ctx->auxil, &chunk->values, 4);
pcc_capture_table__resize(ctx->auxil, &chunk->capts, 0);
{
const int p = ctx->pos;
@@ -2509,7 +2578,7 @@ static pcc_thunk_chunk_t *pcc_evaluate_rule_value(mstro_csq_context_t *ctx) {
L0004:;
}
{
- pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_0, 3, 0);
+ pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_0, 4, 0);
thunk->data.leaf.values.buf[0] = &(chunk->values.buf[0]);
thunk->data.leaf.capt0.range.start = chunk->pos;
thunk->data.leaf.capt0.range.end = ctx->pos;
@@ -2535,7 +2604,7 @@ static pcc_thunk_chunk_t *pcc_evaluate_rule_value(mstro_csq_context_t *ctx) {
L0007:;
}
{
- pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_1, 3, 0);
+ pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_1, 4, 0);
thunk->data.leaf.values.buf[1] = &(chunk->values.buf[1]);
thunk->data.leaf.capt0.range.start = chunk->pos;
thunk->data.leaf.capt0.range.end = ctx->pos;
@@ -2552,7 +2621,7 @@ static pcc_thunk_chunk_t *pcc_evaluate_rule_value(mstro_csq_context_t *ctx) {
}
L0009:;
}
- if (!pcc_apply_rule(ctx, pcc_evaluate_rule_identifier, &chunk->thunks, &(chunk->values.buf[2]))) goto L0008;
+ if (!pcc_apply_rule(ctx, pcc_evaluate_rule_numeric_val, &chunk->thunks, &(chunk->values.buf[2]))) goto L0008;
{
int i;
for (i = 0;; i++) {
@@ -2561,7 +2630,7 @@ static pcc_thunk_chunk_t *pcc_evaluate_rule_value(mstro_csq_context_t *ctx) {
L0010:;
}
{
- pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_2, 3, 0);
+ pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_2, 4, 0);
thunk->data.leaf.values.buf[2] = &(chunk->values.buf[2]);
thunk->data.leaf.capt0.range.start = chunk->pos;
thunk->data.leaf.capt0.range.end = ctx->pos;
@@ -2569,6 +2638,32 @@ static pcc_thunk_chunk_t *pcc_evaluate_rule_value(mstro_csq_context_t *ctx) {
}
goto L0001;
L0008:;
+ ctx->pos = p;
+ pcc_thunk_array__revert(ctx->auxil, &chunk->thunks, n);
+ {
+ int i;
+ for (i = 0;; i++) {
+ if (!pcc_apply_rule(ctx, pcc_evaluate_rule_space, &chunk->thunks, NULL)) goto L0012;
+ }
+ L0012:;
+ }
+ if (!pcc_apply_rule(ctx, pcc_evaluate_rule_identifier, &chunk->thunks, &(chunk->values.buf[3]))) goto L0011;
+ {
+ int i;
+ for (i = 0;; i++) {
+ if (!pcc_apply_rule(ctx, pcc_evaluate_rule_space, &chunk->thunks, NULL)) goto L0013;
+ }
+ L0013:;
+ }
+ {
+ pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_value_3, 4, 0);
+ thunk->data.leaf.values.buf[3] = &(chunk->values.buf[3]);
+ thunk->data.leaf.capt0.range.start = chunk->pos;
+ thunk->data.leaf.capt0.range.end = ctx->pos;
+ pcc_thunk_array__add(ctx->auxil, &chunk->thunks, thunk);
+ }
+ goto L0001;
+ L0011:;
ctx->pos = p;
pcc_thunk_array__revert(ctx->auxil, &chunk->thunks, n);
goto L0000;
@@ -2919,6 +3014,149 @@ L0000:;
return NULL;
}
+static pcc_thunk_chunk_t *pcc_evaluate_rule_numeric_val(mstro_csq_context_t *ctx) {
+ pcc_thunk_chunk_t *const chunk = pcc_thunk_chunk__create(ctx->auxil);
+ chunk->pos = ctx->pos;
+ pcc_value_table__resize(ctx->auxil, &chunk->values, 0);
+ pcc_capture_table__resize(ctx->auxil, &chunk->capts, 2);
+ {
+ const int p = ctx->pos;
+ const int n = chunk->thunks.len;
+ {
+ const int p = ctx->pos;
+ int q;
+ {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0003;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(
+ c == '+' ||
+ c == '-'
+ )) goto L0003;
+ ctx->pos++;
+ }
+ L0003:;
+ {
+ const int p = ctx->pos;
+ int i;
+ for (i = 0;; i++) {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0004;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(c >= '0' && c <= '9')) goto L0004;
+ ctx->pos++;
+ }
+ L0004:;
+ if (i < 1) {
+ ctx->pos = p;
+ goto L0002;
+ }
+ }
+ q = ctx->pos;
+ chunk->capts.buf[0].range.start = p;
+ chunk->capts.buf[0].range.end = q;
+ }
+ {
+ pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_numeric_val_0, 0, 2);
+ thunk->data.leaf.capts.buf[0] = &(chunk->capts.buf[0]);
+ thunk->data.leaf.capt0.range.start = chunk->pos;
+ thunk->data.leaf.capt0.range.end = ctx->pos;
+ pcc_thunk_array__add(ctx->auxil, &chunk->thunks, thunk);
+ }
+ goto L0001;
+ L0002:;
+ ctx->pos = p;
+ pcc_thunk_array__revert(ctx->auxil, &chunk->thunks, n);
+ {
+ const int p = ctx->pos;
+ int q;
+ {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0005;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(
+ c == '+' ||
+ c == '-'
+ )) goto L0005;
+ ctx->pos++;
+ }
+ {
+ int i;
+ for (i = 0;; i++) {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0006;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(c >= '0' && c <= '9')) goto L0006;
+ ctx->pos++;
+ }
+ L0006:;
+ }
+ if (
+ pcc_refill_buffer(ctx, 1) < 1 ||
+ ctx->buffer.buf[ctx->pos] != '.'
+ ) goto L0005;
+ ctx->pos++;
+ {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0005;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(
+ c == 'E' ||
+ c == 'e'
+ )) goto L0005;
+ ctx->pos++;
+ }
+ {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0007;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(
+ c == '+' ||
+ c == '-'
+ )) goto L0007;
+ ctx->pos++;
+ }
+ L0007:;
+ {
+ const int p = ctx->pos;
+ int i;
+ for (i = 0;; i++) {
+ char c;
+ if (pcc_refill_buffer(ctx, 1) < 1) goto L0008;
+ c = ctx->buffer.buf[ctx->pos];
+ if (!(c >= '0' && c <= '9')) goto L0008;
+ ctx->pos++;
+ }
+ L0008:;
+ if (i < 1) {
+ ctx->pos = p;
+ goto L0005;
+ }
+ }
+ q = ctx->pos;
+ chunk->capts.buf[1].range.start = p;
+ chunk->capts.buf[1].range.end = q;
+ }
+ {
+ pcc_thunk_t *const thunk = pcc_thunk__create_leaf(ctx->auxil, pcc_action_numeric_val_1, 0, 2);
+ thunk->data.leaf.capts.buf[1] = &(chunk->capts.buf[1]);
+ thunk->data.leaf.capt0.range.start = chunk->pos;
+ thunk->data.leaf.capt0.range.end = ctx->pos;
+ pcc_thunk_array__add(ctx->auxil, &chunk->thunks, thunk);
+ }
+ goto L0001;
+ L0005:;
+ ctx->pos = p;
+ pcc_thunk_array__revert(ctx->auxil, &chunk->thunks, n);
+ goto L0000;
+ L0001:;
+ }
+ return chunk;
+L0000:;
+ pcc_thunk_chunk__destroy(ctx->auxil, chunk);
+ return NULL;
+}
+
static pcc_thunk_chunk_t *pcc_evaluate_rule_space(mstro_csq_context_t *ctx) {
pcc_thunk_chunk_t *const chunk = pcc_thunk_chunk__create(ctx->auxil);
chunk->pos = ctx->pos;
@@ -2994,7 +3232,7 @@ void mstro_csq_destroy(mstro_csq_context_t *ctx) {
#define MIN(x,y) ((x)<(y) ? (x) : (y))
-#line 358 "cdo_sel_parse.peg"
+#line 370 "cdo_sel_parse.peg"
void
mstro_csq_val__describe(const struct mstro_csq_val *v, size_t indent)
{
diff --git a/maestro/cdo_sel_parse.peg b/maestro/cdo_sel_parse.peg
index 8918fb0486af94ea7957b1f2027ab0e35d421146..8360d84e731726e0f8172c28d1cacf0d1b4d08b1 100644
--- a/maestro/cdo_sel_parse.peg
+++ b/maestro/cdo_sel_parse.peg
@@ -294,9 +294,10 @@ attribute_key <- ide:identifier { $$=ide; DEBUG("ide\n"); }
# values will be stored as strings and parsed with type info later
-value <- space* r:regex space* { $$=r; DEBUG("regex value\n");}
- / space* s:string space* { $$=s; DEBUG("string value\n");}
- / space* i:identifier space* { $$=i; DEBUG("ide value\n"); }
+value <- space* r:regex space* { $$=r; DEBUG("regex value\n");}
+ / space* s:string space* { $$=s; DEBUG("string value\n");}
+ / space* n:numeric_val space* { $$=n; DEBUG("numeric value\n");}
+ / space* i:identifier space* { $$=i; DEBUG("ide value\n"); }
regex <- r:regex_i { $$ = r; DEBUG("CI regex\n"); }
/ r:simple_regex { $$ = r; DEBUG("simple regex\n"); }
@@ -337,7 +338,7 @@ string <- ["] < [^"]* > ["] {
$$=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
$$->strval = strdup($1);
if($$->strval==NULL) {
- ERR("Failed to allocate string value (for regex) parser node\n");
+ ERR("Failed to allocate string value (for string) parser node\n");
abort();
}
}
@@ -345,12 +346,29 @@ string <- ["] < [^"]* > ["] {
$$=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
$$->strval = strdup($2);
if($$->strval==NULL) {
- ERR("Failed to allocate string value (for regex) parser node\n");
+ ERR("Failed to allocate string value (for string) parser node\n");
abort();
}
}
-
+numeric_val <- < [+-]?[0-9]+ > {
+ // signed integer, stored as string
+ $$=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
+ $$->strval = strdup($1);
+ if($$->strval==NULL) {
+ ERR("Failed to allocate string value (for integral) parser node\n");
+ abort();
+ }
+ }
+ / < [+-][0-9]*[.][Ee][+-]?[0-9]+ > {
+ // exponential notation
+ $$=mstro_csq_val__alloc(MSTRO_CSQ_STRING);
+ $$->strval = strdup($2);
+ if($$->strval==NULL) {
+ ERR("Failed to allocate string value (for exponential numeric) parser node\n");
+ abort();
+ }
+ }
## isspace()
space <- [ \t\n\v\f\r] { ; }
@@ -366,7 +384,7 @@ EOF <- !.
#define MIN(x,y) ((x)<(y) ? (x) : (y))
-#line 358 "cdo_sel_parse.peg"
+#line 370 "cdo_sel_parse.peg"
void
mstro_csq_val__describe(const struct mstro_csq_val *v, size_t indent)
{
diff --git a/maestro/core.c b/maestro/core.c
index 19247f2f232df77b115c4025da65e06b378cb433..8837e2aee3f52432201207879553e54466a62fae 100644
--- a/maestro/core.c
+++ b/maestro/core.c
@@ -53,6 +53,10 @@
#include "maestro/i_state.h"
#include "transport/transport.h"
+#ifdef HAVE_NUMA_H
+#include <numa.h>
+#endif
+
/* simplify logging */
#define DEBUG(...) LOG_DEBUG(MSTRO_LOG_MODULE_CORE,__VA_ARGS__)
@@ -80,6 +84,26 @@ mstro_i_destroy_initdata(void* initdata)
return;
}
+static inline
+mstro_status
+mstro_core__numa_init(void)
+{
+#ifdef HAVE_NUMA
+ g_have_numa = (numa_available()==-1 ? false : true);
+ if(g_have_numa) {
+ DEBUG("NUMA: available with %d/%d nodes, %d/%d cpus\n",
+ numa_num_task_nodes(), numa_num_configured_nodes(),
+ numa_num_task_cpus(), numa_num_configured_cpus());
+ /* FIXME: consider initializing some global data structures with this info */
+ } else {
+ DEBUG("NUMA: not available on system\n");
+ }
+#else
+ g_have_numa = false;
+ DEBUG("NUMA: not supported\n");
+#endif
+ return MSTRO_OK;
+}
static inline
mstro_status
@@ -145,6 +169,132 @@ BAILOUT:
/** minimum mlock() limit */
#define MSTRO_MIN_MEMLOCK (4*sizeof(g_component_descriptor))
+
+mstro_status mstro_core_init__setup_schemata(void)
+{
+
+ mstro_status status = MSTRO_OK;
+
+ char * env_schema_list = getenv(MSTRO_ENV_SCHEMA_LIST);
+ char * env_schema_path = getenv(MSTRO_ENV_SCHEMA_PATH);
+ //check that neither are null ...if null make them empty strings
+ if(env_schema_list == NULL)
+ {
+ env_schema_list = "";
+ }
+ else
+ {
+ INFO("List of user attributes schemata %s \n", env_schema_list);
+ strncpy(g_component_descriptor.schema_list, env_schema_list, MSTRO_WORKFLOW_NAME_MAX-1);
+ g_component_descriptor.schema_list[MSTRO_WORKFLOW_NAME_MAX-1] = '\0';
+ }
+
+ if(env_schema_path == NULL)
+ {
+ env_schema_path = "";
+ }
+
+ char *schema_list_token;
+ char *schema_path_token;
+
+
+ // parse and merge the builtin schemas first
+ DEBUG("Parsing Maestro core schema\n");
+ status=mstro_schema_parse(MSTRO_SCHEMA_BUILTIN_YAML_CORE,
+ MSTRO_SCHEMA_BUILTIN_YAML_CORE_LEN,
+ &g_mstro_core_schema_instance);
+ if(status!=MSTRO_OK) {
+ ERR("Failed to parse built-in core schema\n");
+ return status;
+ }
+
+ DEBUG("Parsing ECMWF schema\nFIXME for 0.3.0 ... ECMWF schema shoud not be loaded by default\n");
+ mstro_schema ecmwf;
+ status=mstro_schema_parse(MSTRO_SCHEMA_BUILTIN_YAML_ECMWF,
+ MSTRO_SCHEMA_BUILTIN_YAML_ECMWF_LEN,
+ &ecmwf);
+ if(status!=MSTRO_OK) {
+ ERR("Failed to parse built-in ecmwf schema\n");
+ return status;
+ }
+ status=mstro_schema_merge(g_mstro_core_schema_instance, ecmwf);
+ if(status!=MSTRO_OK) {
+ ERR("Failed to merge core and ECMWF schema\n");
+ return status;
+ }
+
+ // start reading user-defined schemas and merge them.
+ char *end_list_token;
+ char *end_path_token;
+ char *schema_full_path;
+ char *env_schema_path_plus_default;
+
+ // the lenght of all paths = length of paths exported by user + separator + default path "."
+ int path_len = strlen(env_schema_path) + 2 + 2;
+ env_schema_path_plus_default = (char *) malloc(path_len*sizeof(char));
+
+ /* get the first schema */
+ schema_list_token = strtok_r(env_schema_list, SCHEMA_LIST_SEP, &end_list_token);
+ DEBUG("first schema_list_token: %s \n", schema_list_token);
+
+ /* walk through other tokens */
+ while( schema_list_token != NULL )
+ {
+ // parse a schema from the user
+ mstro_schema user_schema;
+ DEBUG("looking for schema_list_token: %s \n", schema_list_token);
+
+ // creating list of paths to visit. i.e. current directory "." + user defined paths
+ snprintf(env_schema_path_plus_default, path_len, "%s%s%s", ".", SCHEMA_PATH_SEP, env_schema_path);
+ DEBUG("list of paths for schemas: %s\n", env_schema_path_plus_default);
+ /* get the first path */
+ schema_path_token = strtok_r(env_schema_path_plus_default, SCHEMA_PATH_SEP, &end_path_token);
+ DEBUG("first schema_path_token: %s\n", schema_path_token);
+
+ /* walk through other paths */
+ while( schema_path_token != NULL )
+ {
+ // forming the full path: path token + / + list name + '\0'
+ int schema_full_path_len = strlen(schema_path_token) + 2 + strlen(schema_list_token) + 2;
+ schema_full_path = (char *) malloc(sizeof(char)*schema_full_path_len);
+ snprintf(schema_full_path, schema_full_path_len, "%s%s%s", schema_path_token, "/", schema_list_token);
+ DEBUG("Parsing user-defined schema from %s\n", schema_full_path);
+ status=mstro_schema_parse_from_file(schema_full_path, &user_schema);
+
+ if(status==MSTRO_OK) {
+ DEBUG("user-defined schema is read from %s\n", schema_full_path);
+ free(schema_full_path);
+ schema_full_path = NULL;
+ break; // no need to try other paths
+ }
+
+ free(schema_full_path);
+ schema_full_path = NULL;
+ // read the next path
+ schema_path_token = strtok_r(NULL, SCHEMA_PATH_SEP, &end_path_token);
+
+ }
+
+ if(status!=MSTRO_OK) {
+ ERR("Failed to parse user_schema from file: %s \n", schema_list_token);
+ return status;
+ }
+ // merge the schema
+ DEBUG("Merging user schema: %s\n", schema_list_token);
+ status=mstro_schema_merge(g_mstro_core_schema_instance, user_schema);
+
+
+ if(status!=MSTRO_OK) {
+ ERR("Failed to merge core and user schema from file %s\n", schema_list_token);
+ return status;
+ }
+ // read the next schema name
+ schema_list_token = strtok_r(NULL, SCHEMA_LIST_SEP, &end_list_token);
+ }
+
+ return status;
+}
+
mstro_status
mstro_core_init(const char *workflow_name,
const char *component_name,
@@ -189,26 +339,55 @@ mstro_core_init(const char *workflow_name,
g_component_descriptor.component_name[MSTRO_WORKFLOW_NAME_MAX-1] = '\0';
strcpy(g_component_descriptor.version, mstro_version());
- status=mstro_schema_parse(MSTRO_SCHEMA_BUILTIN_YAML_CORE,
- MSTRO_SCHEMA_BUILTIN_YAML_CORE_LEN,
- &g_mstro_core_schema_instance);
- if(status!=MSTRO_OK) {
- ERR("Failed to parse built-in core schema\n");
- goto BAILOUT;
- }
- /* FIXME: this should not be here */
- DEBUG("Including ECMWF schema\n");
- mstro_schema ecmwf;
- status=mstro_schema_parse(MSTRO_SCHEMA_BUILTIN_YAML_ECMWF,
- MSTRO_SCHEMA_BUILTIN_YAML_ECMWF_LEN,
- &ecmwf);
- if(status!=MSTRO_OK) {
- ERR("Failed to parse built-in ecmwf schema\n");
- goto BAILOUT;
+ /* check GFS path setting */
+ {
+ char *gfs_dir = getenv(MSTRO_ENV_TRANSPORT_GFS_DIR);
+ if(gfs_dir!=NULL) {
+ bool need_sep = false;
+ size_t l1 = strlen(gfs_dir);
+ assert(l1>0);
+ if(gfs_dir[l1-1]!='/') {
+ need_sep=true;
+ }
+ size_t l2 = strlen(data->workflow_name);
+ size_t l3 = strlen(data->component_name);
+
+ g_mstro_transport_gfs_dir = malloc(l1 + (need_sep? 1 : 0)
+ + l2 + 1
+ + l3 + 1 +1);
+ if(g_mstro_transport_gfs_dir==NULL) {
+ ERR("Failed to allocate GFS transport dir pathname\n");
+ return MSTRO_NOMEM;
+ }
+ size_t pos = 0;
+ strcpy(g_mstro_transport_gfs_dir, gfs_dir);
+ pos += l1;
+ if(need_sep) {
+ g_mstro_transport_gfs_dir[pos] = '/';
+ pos++;
+ }
+ strcpy(g_mstro_transport_gfs_dir+pos, data->workflow_name);
+ pos += l2;
+
+ g_mstro_transport_gfs_dir[pos] = '/';
+ pos++;
+
+ strcpy(g_mstro_transport_gfs_dir+pos, data->component_name);
+ pos += l3;
+ g_mstro_transport_gfs_dir[pos] = '/';
+ pos++;
+ g_mstro_transport_gfs_dir[pos] = '\0';
+ g_mstro_transport_gfs_dir_len = pos;
+ assert(pos==strlen(g_mstro_transport_gfs_dir));
+
+ DEBUG("Set GFS transport directory to %s\n",
+ g_mstro_transport_gfs_dir);
+ }
}
- status=mstro_schema_merge(g_mstro_core_schema_instance, ecmwf);
+
+ /*Reading and integrating attribute schemas*/
+ status = mstro_core_init__setup_schemata();
if(status!=MSTRO_OK) {
- ERR("Failed to merge core and ECMWF schema\n");
goto BAILOUT;
}
@@ -230,6 +409,12 @@ mstro_core_init(const char *workflow_name,
}
pthread_mutex_unlock(&g_initdata_mtx);
+ status = mstro_core__numa_init();
+ if(status!=MSTRO_OK) {
+ ERR("Failed to initialize NUMA support infrastructure\n");
+ goto BAILOUT;
+ }
+
status = mstro_core__mamba_init();
if(status!=MSTRO_OK) {
goto BAILOUT;
diff --git a/maestro/globals.c b/maestro/globals.c
index 6bb1045630c41ba5f26cc102a81dcc186e80fa33..780928d3abbed2864bb55d7e7f01aa507d1fe4bd 100644
--- a/maestro/globals.c
+++ b/maestro/globals.c
@@ -85,3 +85,15 @@ mstro_schema g_mstro_core_schema_instance = NULL;
bool g_have_pool_manager=false;
union mstro_component_descriptor g_component_descriptor;
+
+
+/** the precomputed default path for GFS transport (incl. trailing '/') */
+char *g_mstro_transport_gfs_dir = "./"; /* possibly overwritten at init
+ * time from env*/
+
+/** string length of @ref g_mstro_transport_gfs_dir */
+size_t g_mstro_transport_gfs_dir_len = 2; /* possibly overwritten at
+ * init time */
+
+/** NUMA supported and working? */
+bool g_have_numa = false;
diff --git a/maestro/groups.c b/maestro/groups.c
index 16d4862aa5903f7652d2d8d555690fa7142267fa..6a7f7cdd65437d85217eec66d871351138520aa1 100644
--- a/maestro/groups.c
+++ b/maestro/groups.c
@@ -204,7 +204,7 @@ mstro_group_declare(const char *name, mstro_group* group)
if(s==MSTRO_OK) {
bool trueval = true;
s = mstro_cdo_attribute_set((*group)->group_cdo,
- MSTRO_ATTR_CORE_CDO_ISGROUP, &trueval);
+ MSTRO_ATTR_CORE_CDO_ISGROUP, &trueval, true);
if(s==MSTRO_OK) {
DEBUG("Declared CDO group |%s|\n", name);
(*group)->state = MSTRO_GROUP_STATE_DECLARED;
@@ -312,20 +312,21 @@ mstro_group_seal(mstro_group g)
HASH_ITER(hh, g->members, el, tmp) {
m->declared_members[i] = malloc(sizeof(Mstro__Pool__UUID));
/* need to ensure the GID is ready */
- mstro_status s = mstro_cdo_seal(el->entry);
- if(s!=MSTRO_OK) {
- ERR("Failed to seal declared group member: %d\n", s);
- }
- if(s!=MSTRO_OK || m->declared_members[i]==NULL) {
- for(size_t j=i; j>0; j--) {
- free(m->declared_members[j-1]);
- };
- free(m->declared_members);
- m->declared_members=NULL;
- goto BAILOUT_FREE;
-
+ if(!mstro_cdo_state_check(el->entry, MSTRO_CDO_STATE_SEALED)) {
+ mstro_status s = mstro_cdo_seal(el->entry);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to seal declared group member: %d\n", s);
+ }
+ if(s!=MSTRO_OK || m->declared_members[i]==NULL) {
+ for(size_t j=i; j>0; j--) {
+ free(m->declared_members[j-1]);
+ };
+ free(m->declared_members);
+ m->declared_members=NULL;
+ goto BAILOUT_FREE;
+
+ }
}
-
mstro__pool__uuid__init(m->declared_members[i]);
m->declared_members[i]->qw0 = el->entry->gid.qw[0];
m->declared_members[i]->qw1 = el->entry->gid.qw[1];
@@ -374,7 +375,7 @@ mstro_group_seal(mstro_group g)
s = mstro_cdo_attribute_set(g->group_cdo,
MSTRO_ATTR_CORE_CDO_GROUP_MEMBERS,
- blob);
+ blob, false);
if(s!=MSTRO_OK)
goto BAILOUT_FREE;
diff --git a/maestro/i_maestro_numa.h b/maestro/i_maestro_numa.h
new file mode 100644
index 0000000000000000000000000000000000000000..001029f09d8b6c78e09cb0053be64f84ee9ca219
--- /dev/null
+++ b/maestro/i_maestro_numa.h
@@ -0,0 +1,88 @@
+/* -*- mode:c -*- */
+/** @file
+ ** @brief Maestro NUMA abstraction
+ **/
+/*
+ * Copyright (C) 2021 HPE Switzerland GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MAESTRO_I_NUMA_H_
+#define MAESTRO_I_NUMA_H_ 1
+
+/**@addtogroup MSTRO_Internal
+ **@{
+ **/
+
+/**@defgroup MSTRO_I_NUMA Maestro NUMA abstraction
+ **@{
+ **
+ **/
+
+#ifdef HAVE_NUMAIF_H
+#include <numaif.h>
+#endif
+
+#include "maestro/logging.h"
+
+
+/** obtain node ID for a given address.
+ *
+ * Will always succeed, yielding '0' if NUMA is not supported or ID cannot be obtained */
+static inline
+int
+mstro_numa__node_for_addr(void * addr)
+{
+#ifdef HAVE_NUMA
+ if(g_have_numa) {
+ int node = -1;
+ long s = get_mempolicy(&node,
+ NULL /* node mask */,
+ 0 /* max nodes in mask */,
+ addr, MPOL_F_NODE | MPOL_F_ADDR);
+ if(s<0) {
+ LOG_WARN(MSTRO_LOG_MODULE_CORE,
+ "Failed to obtain NUMA info for addr %p: %ld (%s)\n",
+ addr, s, strerror(-s));
+ return 0;
+ } else {
+ return node;
+ }
+ } else {
+ return 0;
+ }
+#else
+ return 0;
+#endif
+}
+/**@} (end of group MSTRO_I_NUMA) */
+/**@} (end of group MSTRO_Internal) */
+
+#endif /* MAESTRO_I_NUMA_H_ */
+
diff --git a/maestro/ofi.c b/maestro/ofi.c
index 72a1572df8ded128f7f66375d169c9eb9b56f951..06f942cdfd4b5021ee4b5e27611272ecb0bf5992 100644
--- a/maestro/ofi.c
+++ b/maestro/ofi.c
@@ -5,21 +5,21 @@
/*
* Copyright (C) 2019 Cray Computer GmbH
- *
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
- *
+ *
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
- *
+ *
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
- *
+ *
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software without
* specific prior written permission.
- *
+ *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -147,9 +147,9 @@ mstro_ep_desc_create_ofi(mstro_endpoint_descriptor *result_p,
return MSTRO_INVARG;
enum mstro_endpoint_type ept = MSTRO_EP_INVALID;
-
+
switch(fi->addr_format) {
- /** This is the list documented for libfabric 1.5 */
+ /** This is the list documented for libfabric 1.5 */
case FI_SOCKADDR_IN:
ept = MSTRO_EP_OFI_IN4; break;
case FI_SOCKADDR_IN6:
@@ -268,13 +268,13 @@ mstro_ep_desc_serialize__in4(char *dst,
const struct sockaddr_in *sin)
{
mstro_status stat = MSTRO_UNIMPL;
-
+
/* struct sockaddr_in is defined platform-independent and
* already using network byte order as needed, so we can handle
* it as a struct */
tpl_node *tns;
tpl_bin tb;
-
+
tb.sz = sizeof(struct sockaddr_in);
tb.addr = sin;
tns = tpl_map("B", &tb);
@@ -318,7 +318,7 @@ mstro_ep_desc_deserialize__in4(struct sockaddr_in *sin,
char *b64_strval)
{
mstro_status stat=MSTRO_UNIMPL;
-
+
tpl_node *tn;
tpl_bin tb;
tn = tpl_map( "B", &tb );
@@ -333,7 +333,7 @@ mstro_ep_desc_deserialize__in4(struct sockaddr_in *sin,
goto BAILOUT;
}
/* DEBUG("b64decode: buf of length %zu\n", buflen); */
-
+
tpl_load( tn, TPL_MEM, buf, buflen);
tpl_unpack( tn, 0 );
tpl_free(tn);
@@ -361,7 +361,7 @@ mstro_ep_desc_serialize__in6(char *dst,
tpl_bin tb;
INFO("sockaddr_in6 with port %d\n", ntohs(sin->sin6_port));
-
+
tb.sz = sizeof(struct sockaddr_in6);
tb.addr = sin;
tns = tpl_map("B", &tb);
@@ -388,7 +388,7 @@ mstro_ep_desc_serialize__in6(char *dst,
stat=MSTRO_FAIL;
goto BAILOUT;
}
-
+
stat=MSTRO_OK;
BAILOUT:
/* clean up */
@@ -422,7 +422,7 @@ mstro_ep_desc_deserialize__in6(struct sockaddr_in6 *sin,
tpl_load( tn, TPL_MEM, buf, buflen);
tpl_unpack( tn, 0 );
tpl_free(tn);
-
+
assert(tb.sz==sizeof(struct sockaddr_in6));
memcpy(sin, tb.addr, tb.sz);
@@ -455,13 +455,13 @@ mstro_ep_desc_serialize(char **result_p,
struct serialized_endpoint_element serialized_element;
assert(MSTRO_EP__MAX<=INT_MAX);
-
+
tn = tpl_map("A(S(IssUUc#))", &serialized_element, MSTRO_OFI_KEY_LEN_MAX);
char *strval = alloca(MSTRO_EP_STRING_MAX*sizeof(char));
if(strval==NULL)
return MSTRO_NOMEM;
-
+
LL_FOREACH(epd,elt) {
/* each one should serialize into STRVAL buffer, base64-encoded,
* NULL-terminated */
@@ -495,7 +495,7 @@ mstro_ep_desc_serialize(char **result_p,
return MSTRO_FAIL;
}
/* b64 encode */
-
+
size_t needed;
encoded = base64_encode(buf, buflen, &needed);
if(needed>MSTRO_EP_STRING_MAX) {
@@ -513,7 +513,7 @@ mstro_ep_desc_serialize(char **result_p,
stat=MSTRO_OK;
BAILOUT_1:
/* clean up */
- NFREE(encoded);
+ NFREE(encoded);
tpl_free(tns);
free(buf);
} else if(elt->type == MSTRO_EP_OFI_STR) {
@@ -549,7 +549,7 @@ mstro_ep_desc_serialize(char **result_p,
stat=MSTRO_NOMEM;
goto BAILOUT_2;
}
-
+
stat=MSTRO_OK;
BAILOUT_2:
NFREE(encoded);
@@ -558,7 +558,7 @@ mstro_ep_desc_serialize(char **result_p,
elt->type, mstro_ep_descriptor_names[elt->type]);
return MSTRO_FAIL;
}
-
+
serialized_element.type = elt->type;
serialized_element.strval=strdup(strval); /* shrink wrap size */
if(serialized_element.strval==NULL)
@@ -566,7 +566,7 @@ mstro_ep_desc_serialize(char **result_p,
mstro_drc_get_oob_string(&serialized_element.oob_cookie, g_drc_info);
DEBUG("Added OOB info %s\n", serialized_element.oob_cookie);
-
+
/* keys */
serialized_element.info_addr = ep->component_info_addr;
serialized_element.info_keysize = ep->component_info_keysize;
@@ -582,7 +582,7 @@ mstro_ep_desc_serialize(char **result_p,
free(serialized_element.strval);
free(serialized_element.oob_cookie);
}
-
+
/* now write all to a string */
size_t len;
void *buf=NULL;
@@ -619,7 +619,7 @@ mstro_ep_desc_serialize(char **result_p,
BAILOUT:
if(buf)
free(buf);
-
+
return stat;
}
@@ -629,7 +629,7 @@ mstro_ofi_pm_info(char **result_p)
if(result_p==NULL)
return MSTRO_INVOUT;
mstro_status status = MSTRO_OK;
-
+
unsigned char *encoded=NULL;
if(g_endpoints==NULL) {
ERR("No endpoints configured -- did you call mstro_pm_start()?\n");
@@ -645,14 +645,14 @@ mstro_ofi_pm_info(char **result_p)
struct serialized_endpoint_element serialized_element;
assert(MSTRO_EP__MAX<=INT_MAX);
-
+
tn = tpl_map("A(S(IssUUc#))", &serialized_element, MSTRO_OFI_KEY_LEN_MAX);
char *strval = alloca(MSTRO_EP_STRING_MAX*sizeof(char));
if(strval==NULL)
return MSTRO_NOMEM;
mstro_status stat;
-
+
LL_FOREACH(&(g_endpoints->eps[0]),elt) {
mstro_endpoint_descriptor d = elt->descr;
/* each one should serialize into STRVAL buffer, base64-encoded,
@@ -686,7 +686,7 @@ mstro_ofi_pm_info(char **result_p)
return MSTRO_FAIL;
}
/* b64 encode */
-
+
size_t needed;
encoded = base64_encode(buf, buflen, &needed);
if(needed>MSTRO_EP_STRING_MAX) {
@@ -704,7 +704,7 @@ mstro_ofi_pm_info(char **result_p)
stat=MSTRO_OK;
BAILOUT_1:
/* clean up */
- NFREE(encoded);
+ NFREE(encoded);
tpl_free(tns);
free(buf);
} else if(d->type == MSTRO_EP_OFI_STR) {
@@ -740,7 +740,7 @@ mstro_ofi_pm_info(char **result_p)
stat=MSTRO_NOMEM;
goto BAILOUT_2;
}
-
+
stat=MSTRO_OK;
BAILOUT_2:
NFREE(encoded);
@@ -749,7 +749,7 @@ mstro_ofi_pm_info(char **result_p)
d->type, mstro_ep_descriptor_names[d->type]);
return MSTRO_FAIL;
}
-
+
serialized_element.type = d->type;
serialized_element.strval=strdup(strval); /* shrink wrap size */
if(serialized_element.strval==NULL)
@@ -804,7 +804,7 @@ mstro_ofi_pm_info(char **result_p)
BAILOUT:
free(buf);
return status;
-
+
}
@@ -821,7 +821,7 @@ mstro_ep_desc_deserialize__str(char **dst,
mstro_status stat=MSTRO_UNIMPL;
-
+
size_t buflen;
unsigned char *buf = base64_decode((unsigned char*)b64_strval,
strlen(b64_strval),
@@ -858,7 +858,7 @@ mstro_ep_desc_deserialize__uint64(uint64_t (*dst)[6],
}
mstro_status stat=MSTRO_UNIMPL;
-
+
size_t buflen;
unsigned char *buf = base64_decode((unsigned char*)b64_strval,
strlen(b64_strval),
@@ -880,7 +880,7 @@ mstro_ep_desc_deserialize__uint64(uint64_t (*dst)[6],
BAILOUT:
NFREE(buf);
- return stat;
+ return stat;
}
@@ -893,7 +893,7 @@ mstro_ep_desc_deserialize(mstro_endpoint_descriptor *result_p,
tpl_node *tn=NULL;
mstro_endpoint_descriptor *next = NULL;
- if(result_p==NULL)
+ if(result_p==NULL)
return MSTRO_INVOUT;
if(serialized_eps==NULL)
return MSTRO_INVARG;
@@ -903,7 +903,7 @@ mstro_ep_desc_deserialize(mstro_endpoint_descriptor *result_p,
unsigned char *buf = base64_decode((unsigned char*)serialized_eps,
strlen(serialized_eps),
&buflen);
-
+
if(buf==NULL) {
stat=MSTRO_NOMEM;
goto BAILOUT;
@@ -937,9 +937,9 @@ mstro_ep_desc_deserialize(mstro_endpoint_descriptor *result_p,
target->oob_cookie=NULL;
DEBUG("no cookie\n");
}
-
+
target->name=NULL;
-
+
if(eptype==MSTRO_EP_OFI_IN4) {
stat = mstro_ep_desc_deserialize__in4(&(target->in4),
serialized_element.strval);
@@ -956,11 +956,11 @@ mstro_ep_desc_deserialize(mstro_endpoint_descriptor *result_p,
|| eptype==MSTRO_EP_OFI_MLX) {
/** transparently handles one .. six entries */
stat = mstro_ep_desc_deserialize__uint64(&target->gni,
- serialized_element.strval);
+ serialized_element.strval);
} else if(eptype==MSTRO_EP_OFI_STR) {
target->str=NULL;
stat = mstro_ep_desc_deserialize__str(&(target->str),
- serialized_element.strval);
+ serialized_element.strval);
} else {
ERR("Unsupported EP type: %d\n", eptype);
stat = MSTRO_UNIMPL;
@@ -994,7 +994,7 @@ mstro_ep_desc_deserialize(mstro_endpoint_descriptor *result_p,
}
BAILOUT:
free(buf);
-
+
return stat;
}
@@ -1090,7 +1090,7 @@ mstro_ep_desc_describe(mstro_endpoint_descriptor desc)
s, errno, strerror(errno));
abort();
}
-
+
desc->name = buf;
}
}
@@ -1107,7 +1107,7 @@ mstro_ep_desc_to_ofi_addr(uint32_t *addr_format,
return MSTRO_INVOUT;
if(epd==NULL)
return MSTRO_INVARG;
-
+
switch(epd->type) {
case MSTRO_EP_OFI_IN4:
*addr_format = FI_SOCKADDR_IN;
@@ -1167,7 +1167,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
return MSTRO_INVOUT;
if(fi==NULL)
return MSTRO_INVARG;
-
+
struct fid_fabric *fabric = NULL;
struct fi_eq_attr eq_attr;
struct fid_eq *eq = NULL;
@@ -1178,7 +1178,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
struct fi_cq_attr cq_attr;
struct fid_cq *cq = NULL;
uint8_t *mr_key = NULL;
-
+
/* create fabric object */
stat = fi_fabric(fi->fabric_attr, &fabric, NULL);
if(stat!=0) {
@@ -1195,15 +1195,15 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
* allocated early on in ofi_init (on pool manager) or obtained
* from PM_INFO (on clients) */
assert(g_drc_info!=NULL);
-
+
stat = mstro_drc_insert_ofi(fi, g_drc_info);
if(stat!=MSTRO_OK) {
ERR("Failed to insert DRC credential into fabric info\n");
retstat=MSTRO_FAIL;
goto BAILOUT_FAIL;
- }
+ }
}
-
+
/* create domain */
stat = fi_domain(fabric, fi, &domain, NULL);
if(stat!=0) {
@@ -1232,7 +1232,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
ERR("fi_cq_open failed: %d (%s)\n", stat, fi_strerror(-stat));
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
}
-
+
/* event queue */
/* create event queue */
memset(&eq_attr, 0, sizeof(eq_attr));
@@ -1253,7 +1253,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
}
/* INFO("This is a %s\n", fi_tostr(fi,FI_TYPE_INFO)); */
-
+
/* connectionless endpoint needs address vector */
memset(&av_attr, 0, sizeof(av_attr));
av_attr.type = FI_AV_MAP;
@@ -1262,14 +1262,14 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
ERR("fi_av_open failed: %d (%s)\n", stat, fi_strerror(-stat));
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
}
-
+
/* bind address vector to endpoint */
stat = fi_ep_bind(ep, &av->fid, 0);
if(stat!=0) {
ERR("fi_ep_bind for av failed: %d (%s)\n", stat, fi_strerror(-stat));
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
}
-
+
/* bind event queue to endpoint */
stat = fi_ep_bind(ep, &eq->fid, 0);
if(stat!=0) {
@@ -1280,7 +1280,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
eq=NULL;
} else {
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
- }
+ }
}
/* bind cq to endpoint */
stat = fi_ep_bind(ep, &cq->fid, FI_TRANSMIT|FI_RECV);
@@ -1288,7 +1288,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
ERR("fi_ep_bind for cq failed: %d (%s)\n", stat, fi_strerror(-stat));
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
}
-
+
/* enable! */
stat = fi_enable(ep);
if(stat!=0) {
@@ -1380,7 +1380,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
ERR("Failed to retrieve key for component info memory registration: %d (%s)\n");
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
}
- assert(sizeof(key)<=MSTRO_OFI_KEY_LEN_MAX);
+ assert(sizeof(key)<=MSTRO_OFI_KEY_LEN_MAX);
/* keysize is 4 on verbs ... we try to let the compiler handle byte order issues here by casting */
switch(keysize) {
case 1: {
@@ -1402,7 +1402,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
memcpy(mr_key, &key, sizeof(key));
break;
}
- default:
+ default:
ERR("Unsupported MR key size %d\n", keysize);
assert(0);
}
@@ -1417,13 +1417,13 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
addr=(uint64_t)&g_component_descriptor;
}
}
-
-
+
+
{
uint64_t k = 0;
memcpy(&k, mr_key, keysize); /* just for printing */
-
+
INFO("Component info RDMA block registered%s: local %p, MR %p, addr %" PRIx64 ", desc %p, keysize %zu (key uint64 start: 0x%" PRIx64 ")\n",
fi->domain_attr->mr_mode & FI_MR_RAW ? " (raw keys)" : "",
&g_component_descriptor, mr, addr, fi_mr_desc(mr), keysize, k);
@@ -1440,7 +1440,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
dst->component_info_addr = addr;
dst->component_info_keysize = keysize;
dst->component_info_raw_key = mr_key;
-
+
dst->next = NULL;
@@ -1450,7 +1450,7 @@ mstro_ep_build_from_ofi(struct mstro_endpoint *dst,
ERR("Failed to construct worker mgmt ep description: %d\n", retstat);
goto BAILOUT_FAIL;
}
-
+
retstat = mstro_ep_desc_serialize(&dst->addr_serialized,
dst);
if(retstat!=MSTRO_OK) {
@@ -1483,7 +1483,7 @@ BAILOUT_FAIL:
/* wait sets (none) */
CLEANUP(fabric,"fabric");
#undef CLEANUP
-
+
BAILOUT:
return retstat;
}
@@ -1502,7 +1502,7 @@ mstro_endpoint_describe(struct mstro_endpoint *ep)
char addrbuf[EP_DESC_BUF_MAX];
ep_name_buf[0]='\0';
-
+
size=0;
ret = fi_getname(&ep->ep->fid,NULL,&size);
if(ret!=-FI_ETOOSMALL) {
@@ -1511,16 +1511,16 @@ mstro_endpoint_describe(struct mstro_endpoint *ep)
}
assert(size<EP_DESC_BUF_MAX);
fi_getname(&ep->ep->fid,addrbuf,&size);
-
+
size=0;
fi_av_straddr(ep->av, addrbuf, NULL, &size);
assert(size<EP_DESC_BUF_MAX);
fi_av_straddr(ep->av, addrbuf, strbuf, &size);
-
+
size=snprintf(ep_name_buf, EP_DESC_BUF_MAX,
"OFI EP prov %s name %s straddr %s",
- ep->fi->fabric_attr->prov_name,
+ ep->fi->fabric_attr->prov_name,
ep->fi->fabric_attr->name, strbuf);
assert(size<1024);
DONE:
@@ -1655,7 +1655,7 @@ mstro_ofi__order_fi_list(struct fi_info **fi)
{
if(fi==NULL)
return MSTRO_INVARG;
-
+
struct fi_info *head = *fi;
LL_SORT(head, mstro_ofi__fi_info_cmp);
*fi = head;
@@ -1702,11 +1702,11 @@ mstro_ofi_init(void)
hints->mode = MSTRO_OFI_MODE;
hints->ep_attr->type = MSTRO_OFI_EP_TYPE;
hints->domain_attr->mr_mode = MSTRO_OFI_MRMODE;
-
+
/* we really want 1.8 or above */
stat = fi_getinfo(MSTRO_OFI_VERSION, NULL, NULL, 0, hints, &fi);
fi_freeinfo(hints);
-
+
if(stat!=0) {
ERR("fi_getinfo failed: %d (%s)\n", stat, fi_strerror(-stat));
retstat=MSTRO_FAIL; goto BAILOUT_FAIL;
@@ -1748,7 +1748,7 @@ mstro_ofi_init(void)
goto BAILOUT_FAIL;
}
- /* also do it for the PM descriptor block we'll RDMA read into */
+ /* also do it for the PM descriptor block we'll RDMA read into */
assert(sizeof(g_pm_component_descriptor)%sysconf(_SC_PAGESIZE)==0);
retstat = mstro_memlock(&g_pm_component_descriptor, sizeof(g_pm_component_descriptor));
if(retstat!=MSTRO_OK) {
@@ -1762,11 +1762,11 @@ mstro_ofi_init(void)
LL_FOREACH(fi,tmp) {
NOISE("index %zu\n", i);
i++;
- DEBUG("potential endpoint: %s %s\n", tmp->fabric_attr->prov_name,
+ DEBUG("potential endpoint: %s %s\n", tmp->fabric_attr->prov_name,
tmp->fabric_attr->name);
retstat = mstro_ep_build_from_ofi(&(g_endpoints->eps[g_endpoints->size]),
tmp);
-
+
if(retstat!=MSTRO_OK) {
WARN("Failed to build EP %zu, trying others\n", i);
} else {
@@ -1799,18 +1799,18 @@ mstro_ofi_init(void)
}
/* fix pointer into invalid last entry */
g_endpoints->eps[g_endpoints->size-1].next = NULL;
-
+
if(g_endpoints!=NULL) {
INFO("%zu usable endpoints\n", g_endpoints->size);
retstat = MSTRO_OK;
} else {
ERR("No usable endpoints found\n");
}
-
+
goto BAILOUT;
BAILOUT_FAIL:
; /* FIXME: free resources */
-
+
BAILOUT:
return retstat;
}
@@ -1842,7 +1842,7 @@ mstro_ofi_finalize(void)
}
/* every EP has one lock on the component descriptor */
mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor));
-
+
#define CLOSE_FID(member, descr) do { \
if(e->member) { \
DEBUG("Closing down %s for ep %zu\n", descr, i); \
@@ -1851,7 +1851,7 @@ mstro_ofi_finalize(void)
descr, i, s, fi_strerror(-s)); } \
} \
} while(0)
-
+
CLOSE_FID(ep,"EP");
CLOSE_FID(cq,"CQ");
CLOSE_FID(av,"AV");
@@ -1859,16 +1859,16 @@ mstro_ofi_finalize(void)
CLOSE_FID(domain,"DOMAIN");
// CLOSE_FID(fi,"FABRIC");
#undef CLOSE_FID
-
+
}
}
DEBUG("OFI endpoints closed\n");
-
+
/* It's safe to call this even if we've not yet locked the page; we
* ignore the error that we'd get in this case */
mstro_memunlock(&g_component_descriptor, sizeof(g_component_descriptor));
mstro_memunlock(&g_pm_component_descriptor, sizeof(g_pm_component_descriptor));
-
+
return MSTRO_OK;
}
@@ -1890,7 +1890,7 @@ struct mstro_ofi_msg_context_ {
};
bool has_completion; /**< indicates if pthread waiting
* infrastructure is initialized in this
- * context and should be used*/
+ * context and should be used*/
pthread_cond_t completion; /**< Condition variable for originator of
* message to */
pthread_mutex_t lock; /**< a lock, mostly because cond_wait
@@ -1901,7 +1901,7 @@ typedef struct mstro_ofi_msg_context_ * mstro_ofi_msg_context;
/** Create a message completion context for @arg msg.
-
+
If @arg want_completion is true a mutex and a condition variable
will be initialized and the completion handler can signal
completion on that condition variable to the originator.
@@ -1923,7 +1923,7 @@ mstro_ofi__msg_context_create(mstro_ofi_msg_context *result_p,
/* RDMA ops don't have non-NULL msg */
/* if(msg==NULL) */
/* return MSTRO_INVARG; */
-
+
mstro_ofi_msg_context ctx
= malloc(sizeof(struct mstro_ofi_msg_context_));
if(ctx==NULL) {
@@ -1939,7 +1939,7 @@ mstro_ofi__msg_context_create(mstro_ofi_msg_context *result_p,
free(ctx);
ctx=NULL;
goto BAILOUT;
- }
+ }
res = pthread_cond_init(&ctx->completion, NULL);
if(res!=0) {
ERR("Failed to init ofi msg cond var: %d\n", res);
@@ -1957,7 +1957,7 @@ mstro_ofi__msg_context_create(mstro_ofi_msg_context *result_p,
stat = MSTRO_OK;
BAILOUT:
*result_p = ctx;
- return stat;
+ return stat;
}
/** destroy an OFI message context */
@@ -1975,7 +1975,45 @@ mstro_ofi__msg_context_destroy(mstro_ofi_msg_context ctx)
return MSTRO_OK;
}
-
+/** Check the schema list provided from pm is compatible with the component schema list */
+/* compatible means that pm supports all of the schemas in the component schema list or more */
+/* FIXME check that the schema versions are compatible between the pm and the component */
+bool schema_list_compatible_p(const char * component_schema_list, const char * pm_schema_list)
+{
+ char *schema_name;
+ char *end_list_token;
+
+ if (component_schema_list == NULL)
+ {
+ /*nothing to look for here*/
+ //DEBUG("[schema_list_compatible_p] component schema list is NULL");
+ return true;
+ }
+ else if (pm_schema_list ==NULL) {
+ /*component_schema_list is not empty but pm_schema_list is empty ... we should look no further*/
+ //DEBUG("[schema_list_compatible_p] pm schema list is NULL");
+ return false;
+ }
+ // split the schema names in the component_schema_list
+ schema_name = strtok_r(component_schema_list, SCHEMA_LIST_SEP, &end_list_token);
+ // for each schema name in the component_schema_list ...try to find it in the pm_schema_list
+ while( schema_name != NULL )
+ {
+ DEBUG("Looking for schema %s \n", schema_name);
+ /*if we did not find the component schema name in the pm_schema_list*/
+ if(strstr(pm_schema_list, schema_name) == NULL) {
+ DEBUG("Schema %s not found in pm schema list\n", schema_name);
+ return false;
+ }
+ DEBUG("Found schema %s in %s\n", schema_name, pm_schema_list);
+ // read the next schema name
+ schema_name = strtok_r(NULL, SCHEMA_LIST_SEP, &end_list_token);
+ }
+
+ /*We reach here only if all component schemas were found in the pm_schema_list*/
+ return true;
+
+}
/** select the best match between remote and local endpoints */
mstro_status
@@ -2053,7 +2091,7 @@ mstro_ofi__select_endpoint(struct mstro_endpoint_descriptor_ *remote,
stat = MSTRO_FAIL;
goto BAILOUT;
}
-
+
uint64_t mr_addr=0;
/* since the EP accepted the address we interpolate that
* MR_RAW will be set identically (while we really should be
@@ -2075,7 +2113,7 @@ mstro_ofi__select_endpoint(struct mstro_endpoint_descriptor_ *remote,
}
DEBUG("Checking for PM config block MR at (remote addr) 0x%" PRIx64 ", key %" PRIx64 "\n",
mr_addr, mr_key);
-
+
assert(ctx->msg==NULL);
assert(my_ep->peer_info_mr!=NULL); /* incoming buffer has been registered at local endpoint set creation */
void * local_buf_mr_desc = fi_mr_desc(my_ep->peer_info_mr);
@@ -2123,8 +2161,15 @@ mstro_ofi__select_endpoint(struct mstro_endpoint_descriptor_ *remote,
INFO("Component descriptor mismatch, illegal type or other workflow, skipping\n");
continue;
}
+ DEBUG("PM schema list %s \n", g_pm_component_descriptor.schema_list);
+ DEBUG("Component schema list %s \n", g_component_descriptor.schema_list);
+ if(!schema_list_compatible_p(g_component_descriptor.schema_list, g_pm_component_descriptor.schema_list))
+ {
+ INFO("Component descriptor mismatch, PM does not support the schema list of component, skipping\n");
+ continue;
+ }
} /* end of possible config block verification */
-
+
*remote_addr_p = translated_addr;
*local_p = my_ep;
stat=MSTRO_OK;
@@ -2161,7 +2206,7 @@ mstro_ofi__submit_message(struct mstro_endpoint *ep,
assert(env!=NULL);
DEBUG("Sending message %s, size %zu, %p using ep %p\n",
env->descriptor->name, env->payload_size, env, ep);
-
+
int res;
do {
assert(env!=context); /* catch common mistake: context needs to be a
@@ -2174,7 +2219,7 @@ mstro_ofi__submit_message(struct mstro_endpoint *ep,
/* DEBUG("tsend with tag %"PRIu64"\n", (uint64_t)msg->type); */
} else {
- res = fi_send(ep->ep, env->data, env->payload_size,
+ res = fi_send(ep->ep, env->data, env->payload_size,
descriptor, dst, context);
}
if(res!=0) {
@@ -2205,9 +2250,9 @@ mstro_ofi__submit_message_wait(struct mstro_endpoint *ep, fi_addr_t dst,
ERR("Failed to acquire msg context lock: %d\n", ret);
return MSTRO_FAIL;
}
-
+
stat = mstro_ofi__submit_message(ep, dst, msg, NULL, ctx);
- if(stat!=MSTRO_OK)
+ if(stat!=MSTRO_OK)
return stat;
ret = pthread_cond_wait(&ctx->completion, &ctx->lock);
@@ -2240,8 +2285,8 @@ mstro_ofi__submit_message_nowait(struct mstro_endpoint *ep, fi_addr_t dst,
} else {
;
}
-
-
+
+
/* for non-completion contexts the message handler frees the
* context */
return mstro_ofi__submit_message(ep, dst, msg, NULL, ctx);
@@ -2257,7 +2302,7 @@ mstro_ofi__expect_message_wait(struct fid_ep *ep, fi_addr_t src,
= mstro_ofi__msg_context_create(&ctx, msg, true);
if(stat!=MSTRO_OK)
return stat;
-
+
int ret = pthread_mutex_lock(&ctx->lock);
if(ret!=0) {
ERR("Failed to acquire msg context lock: %d\n", ret);
@@ -2266,7 +2311,7 @@ mstro_ofi__expect_message_wait(struct fid_ep *ep, fi_addr_t src,
/* FIXME: split out do loop like for submit so that we don't need
* fi_recv in arbitrary places anymore */
-
+
int res;
do {
if(MSTRO_MSG_HAS_TAG(msg)) {
@@ -2354,7 +2399,7 @@ mstro_pm__register_app(Mstro__Pool__Join *join_msg,
}
INFO("EPD serial %s\n", join_msg->serialized_endpoint);
INFO("EPD parsed as %s\n", mstro_ep_desc_describe(epd));
-
+
uint32_t addr_format;
void *remote_addr;
size_t addrlen;
@@ -2384,7 +2429,7 @@ mstro_pm__register_app(Mstro__Pool__Join *join_msg,
DEBUG("App %s advertises %zu transport methods\n",
join_msg->component_name, join_msg->transport_methods->n_supported);
-
+
/* insert into registry table */
s = mstro_pm_app_register(ep, translated_addr,
strdup(join_msg->serialized_endpoint),
@@ -2398,7 +2443,7 @@ mstro_pm__register_app(Mstro__Pool__Join *join_msg,
join_msg->transport_methods = NULL; /* we kept a reference to it, caller will free the message */
DEBUG("Assigned app ID %"PRIu64"\n", (*entry_p)->appid);
}
-
+
BAILOUT:
return s;
BAILOUT_FREE:
@@ -2546,24 +2591,24 @@ mstro_ofi__check_and_handle_cq(struct mstro_endpoint *ep,
INFO("Completion of op with ctx %p message %p%s of size %zu on endpoint %p\n",
ctx, ctx->msg, re_post ? " (slot i)" : "",
entry.len, ep);
-
+
if(re_post) {
DEBUG("Posting fresh recv on slot %p", expected_slot_p);
status = mstro_ofi__loop_post_recv(ep->ep, expected_slot_p, expected_ctx_p);
}
-
-
+
+
if(entry.flags&FI_RECV) {
assert(ctx->msg!=NULL);
ctx->msg->payload_size = entry.len; /* this permits us to not pass ofi 'entry' to pc/pm handler */
const struct mstro_msg_envelope *msg = ctx->msg;
status = incoming_msg_handler(msg, ep);
-
+
if(status!=MSTRO_OK) {
ERR("Error handling incoming message, dropping it: %d (%s)\n",
status, mstro_status_description(status));
}
-
+
if(re_post) {
; /* fresh receive will be posted later, and slot messages have no completion watchers */
} else {
@@ -2571,42 +2616,42 @@ mstro_ofi__check_and_handle_cq(struct mstro_endpoint *ep,
DEBUG("non-slot message, cleaning up\n");
if(ctx->has_completion) {
DEBUG("ctx asks for completion notification\n");
- status = mstro_ofi__maybe_notify_completion(ctx);
+ status = mstro_ofi__maybe_notify_completion(ctx);
} else {
DEBUG("ctx wants no completion notification, cleaning up ctx\n");
mstro_msg_envelope_free(ctx->msg);
status = mstro_ofi__msg_context_destroy(ctx);
}
}
-
+
} else if(entry.flags&FI_SEND) {
assert(ctx->msg!=NULL);
struct mstro_msg_envelope *msg = ctx->msg;
-
+
/* completion of our sent messages */
DEBUG("Send of message %p type %s completed\n",
msg, msg->descriptor->name);
/* otherwise clean it up */
if(ctx->has_completion) {
DEBUG("ctx asks for completion notification\n");
- status = mstro_ofi__maybe_notify_completion(ctx);
+ status = mstro_ofi__maybe_notify_completion(ctx);
} else {
DEBUG("ctx wants no completion notification, cleaning up ctx\n");
mstro_msg_envelope_free(msg);
status = mstro_ofi__msg_context_destroy(ctx);
}
-
+
} else if(entry.flags&FI_RMA) {
assert(ctx->msg==NULL);
DEBUG("RDMA op completed, ctx %p\n", ctx);
assert(ctx->has_completion); /* it does not make sense to
* not have this set */
- status = mstro_ofi__maybe_notify_completion(ctx);
+ status = mstro_ofi__maybe_notify_completion(ctx);
} else {
DEBUG("Unexpected completion, flags %ull\n", entry.flags);
}
-
- return status;
+
+ return status;
}
static inline
@@ -2651,15 +2696,15 @@ mstro_ofi__loop(_Atomic bool *terminate,
} while(false==atomic_load(terminate));
DEBUG("PM comm thread quits: loop status %d, global status %d\n",
s, status);
-
+
BAILOUT:
if(slots) {
- for(i=0; i<g_endpoints->size; i++)
+ for(i=0; i<g_endpoints->size; i++)
mstro_msg_envelope_free(slots[i]);
free(slots);
}
if(ctxts) {
- for(i=0; i<g_endpoints->size; i++)
+ for(i=0; i<g_endpoints->size; i++)
mstro_ofi__msg_context_destroy(ctxts[i]);
free(ctxts);
}
@@ -2672,7 +2717,7 @@ mstro_ofi_pm_loop(_Atomic bool *terminate)
g_pool_app_id = MSTRO_APP_ID_MANAGER;
return mstro_ofi__loop(terminate, mstro_pm_handle_msg);
-}
+}
mstro_status
mstro_ofi_pc_loop(_Atomic bool *terminate)
@@ -2756,7 +2801,7 @@ mstro_pc_thread(void *closure)
s, mstro_status_description(s));
}
DEBUG("pool client thread stopping\n");
-
+
return NULL;
}
*/
@@ -2800,9 +2845,9 @@ mstro_pc_terminate(void)
leave.base.descriptor->name);
goto BAILOUT;
}
-
+
status = mstro_pmp_send_nowait(MSTRO_APP_ID_MANAGER, &msg);
-
+
/* The BYE message will be handled by the normal incoming message handler
* We can tell it's done by checking g_mstro_pm_attached */
switch(status) {
@@ -2864,7 +2909,7 @@ mstro_pm_attach(const char *remote_pm_info)
mstro_endpoint_descriptor pm_epd=NULL;
mstro_status s=mstro_ep_desc_deserialize(&pm_epd, remote_pm_info);
-
+
if(s!=MSTRO_OK) {
ERR("Failed to parse pool manager info: %d (%s%s)\n",
s, mstro_status_description(s),
@@ -2881,7 +2926,7 @@ mstro_pm_attach(const char *remote_pm_info)
{
mstro_endpoint_descriptor tmp;
size_t num_cookies = 0;
-
+
LL_FOREACH(pm_epd, tmp) {
INFO("Pool manager endpoint: %s\n", mstro_ep_desc_describe(tmp));
if(tmp->type==MSTRO_EP_OFI_GNI) {
@@ -2900,13 +2945,13 @@ mstro_pm_attach(const char *remote_pm_info)
}
}
}
-
+
s=mstro_ofi_init();
if(s!=MSTRO_OK) {
ERR("Failed to initialize OFI layer\n");
goto BAILOUT;
}
-
+
/* start handler thread: run pool client loop. It will stop when mstro_pc_terminate is called */
/* we need to start it before endpoint selection since we do RDMA
* read for the config block, and that needs to report completion
@@ -2919,30 +2964,30 @@ mstro_pm_attach(const char *remote_pm_info)
goto BAILOUT;
}
g_component_descriptor.type = MSTRO_COMPONENT_TYPE_APP;
-
-
+
+
s=mstro_ofi__select_endpoint(pm_epd, &g_pm_endpoint, &g_pm_addr);
if(s!=MSTRO_OK) {
ERR("Failed to select a suitable endpoint to talk to pool manager\n");
goto BAILOUT;
}
assert(g_pm_endpoint!=NULL);
-
+
/* register pool manager as a pmp app entry */
s = mstro_pm_app_register_manager(g_pm_endpoint, g_pm_addr);
if(s!=MSTRO_OK) {
ERR("Failed to register Pool Manager in registry\n");
goto BAILOUT;
}
-
-
+
+
/* send JOIN */
-
-
+
+
/* FIXME make function set_transport_default() beware protobuf stack init
*/
/* if we have OFI we'll add another entry */
-
+
Mstro__Pool__TransportKind transport_kinds[] =
{ MSTRO__POOL__TRANSPORT_KIND__GFS
#ifdef HAVE_MIO
@@ -2950,18 +2995,18 @@ mstro_pm_attach(const char *remote_pm_info)
#endif
};
size_t num_available_transports = sizeof(transport_kinds)/sizeof(transport_kinds[0]);
-
+
Mstro__Pool__TransportMethods transport_methods
= MSTRO__POOL__TRANSPORT_METHODS__INIT;
transport_methods.n_supported = num_available_transports;
transport_methods.supported
= (Mstro__Pool__TransportKind*)&transport_kinds;
-
+
const char* env_transport_default = getenv(MSTRO_ENV_TRANSPORT_DEFAULT);
if (env_transport_default != NULL) {
int i;
int found = 0;
- for ( i = 0; i < MSTRO__POOL__TRANSPORT_KIND__NUMBER_OF_KINDS; i++ ) {
+ for ( i = 0; i < MSTRO__POOL__TRANSPORT_KIND__NUMBER_OF_KINDS; i++ ) {
if (!strcmp(env_transport_default, g_transport_registry[i].name)) {
transport_methods.supported[0] = (Mstro__Pool__TransportKind)i;
found = 1;
@@ -2978,13 +3023,13 @@ mstro_pm_attach(const char *remote_pm_info)
for (i = 0; i < transport_methods.n_supported; i++ )
INFO("#%d %s \n", i, g_transport_registry[transport_methods.supported[i]].name);
/**/
-
+
Mstro__Pool__Join join = MSTRO__POOL__JOIN__INIT;
join.protocol_version = MSTRO_POOL_PROTOCOL_VERSION;
join.serialized_endpoint = g_pm_endpoint->addr_serialized;
join.transport_methods = &transport_methods;
join.component_name = g_initdata->component_name;
-
+
Mstro__Pool__MstroMsg msg = MSTRO__POOL__MSTRO_MSG__INIT;
s = mstro_pmp_package(&msg, (ProtobufCMessage*)&join);
if(s!=MSTRO_OK) {
@@ -2992,13 +3037,13 @@ mstro_pm_attach(const char *remote_pm_info)
join.base.descriptor->name);
goto BAILOUT;
}
-
+
s = mstro_pmp_send_nowait(MSTRO_APP_ID_MANAGER, &msg);
/* The WELCOME message will be handled by the normal incoming
* message handler. We can tell it's done by checking
* g_pool_app_id */
-
+
switch(s) {
case MSTRO_OK: {
/* spin-wait */
@@ -3010,7 +3055,7 @@ mstro_pm_attach(const char *remote_pm_info)
}
break;
}
- case MSTRO_NO_PM:
+ case MSTRO_NO_PM:
ERR("Confusion: JOIN message failed with 'no PM info' error\n");
/* fall-thru */
default:
@@ -3018,15 +3063,13 @@ mstro_pm_attach(const char *remote_pm_info)
s, mstro_status_description(s));
return s;
}
-
-
+
+
/* Welcome received */
- INFO("Welcome message received, our app id is %"PRIu64"\n",
+ INFO("Welcome message received, our app id is %"PRIu64"\n",
g_pool_app_id);
-
+
BAILOUT:
return s;
}
-
-
diff --git a/maestro/pool.c b/maestro/pool.c
index 201fc667c1b85f8b1b2cae148e364aa05e3626d6..4fd09ec46454ba78be1c2856a735984a6a13c08b 100644
--- a/maestro/pool.c
+++ b/maestro/pool.c
@@ -1816,10 +1816,3 @@ mstro_subscription_dispose(mstro_subscription s)
-mstro_status
-mstro_pool__resolve_cdoid(const struct mstro_cdo_id *id,
- const char **name_p)
-{
- /* we use the subscription infrastructure for this */
- return mstro_pool_resolve_cdoid(id, name_p);
-}
diff --git a/maestro/pool_client.c b/maestro/pool_client.c
index 774d091190795c977ad40f313e881e400fa4d5de..a2e169e20fac4c28f7447de9c442290a02e02338 100644
--- a/maestro/pool_client.c
+++ b/maestro/pool_client.c
@@ -78,15 +78,19 @@ mstro_pc__handle_resolve_reply(const Mstro__Pool__ResolveReply *reply)
}
-/* store @arg new_attributes in the serialized attributes slot of @arg cdo, and replace the attribute dict values accordingly. Consumes @arg new_attributes. */
+/* store @arg new_attributes in the serialized attributes slot of @arg
+ * cdo, and replace the attribute dict values accordingly. Consumes
+ * @arg new_attributes. */
static inline
mstro_status
-mstro_cdo__replace_attributes(mstro_cdo cdo, Mstro__Pool__Attributes *new_attributes)
+mstro_cdo__replace_attributes(mstro_cdo cdo,
+ Mstro__Pool__Attributes *new_attributes)
{
/* We can't lock a CDO on it's own. But this function should only be
- * called before SEAL state change completes, and so users can't be
- * accessing the CDO legally, and it's way before the CDO could be
- * in the pool, so transport etc can't touch it either. */
+ * called before SEAL state change completes, or during WITHDRAW
+ * etc. and so users can't be accessing the CDO legally, and it's
+ * way before or after the CDO could be in the pool, so transport
+ * etc can't touch it either. */
assert(mstro_cdo_state_check(cdo, MSTRO_CDO_STATE_DECLARED));
mstro_status s = MSTRO_UNIMPL;
@@ -269,6 +273,31 @@ mstro_pc__select_transfer_method(mstro_cdo cdo,
return MSTRO_OK;
}
+static inline
+mstro_status
+mstro_pc__construct_gfs_path_for_cdo(const mstro_cdo src_cdo,
+ char **path)
+{
+ mstro_status s=MSTRO_UNIMPL;
+ assert(path!=NULL);
+ assert(src_cdo!=NULL);
+
+ size_t l = g_mstro_transport_gfs_dir_len + strlen(src_cdo->name) + 1;
+ *path = malloc(l);
+ if(*path==NULL) {
+ return MSTRO_NOMEM;
+ }
+ strcpy(*path, g_mstro_transport_gfs_dir);
+
+ /* FIXME: possibly add a (configurable?) number of subdirectories
+ * based on first segment of CDO name */
+
+ strcpy((*path)+g_mstro_transport_gfs_dir_len, src_cdo->name);
+ DEBUG("Constructed GFS transport path %s\n", *path);
+ return MSTRO_OK;
+}
+
+
static inline
mstro_status
mstro_pc__handle_initiate_transfer(const Mstro__Pool__InitiateTransfer* init)
@@ -357,22 +386,28 @@ mstro_pc__handle_initiate_transfer(const Mstro__Pool__InitiateTransfer* init)
Mstro__Pool__TransferTicket ticket = MSTRO__POOL__TRANSFER_TICKET__INIT;
ticket.cdoid = &id;
- const int64_t *size;
- enum mstro_cdo_attr_value_type type;
- if (! (MSTRO_OK == mstro_cdo_attribute_get(src_cdo,
- MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- &type, (const void**)&size))) {
- ERR("Couldn't retrieve CDO %s local_size needed for transport\n", src_cdo->name);
+ mstro_status s=MSTRO_UNIMPL;
+ const void *size=NULL;
+ enum mstro_cdo_attr_value_type vt;
+ s = mstro_attribute_dict_get(src_cdo->attributes,
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &vt, &size, NULL, false);
+ if(s==MSTRO_NOENT && vt==MSTRO_CDO_ATTR_VALUE_INVALID) {
+ ERR("CDO has mamba-array but no local-size\n");
+ return MSTRO_FAIL;
+ }
+ if(s!=MSTRO_OK) {
+ ERR("Failed to retrieve local-size on CDO\n");
return MSTRO_FAIL;
}
- int64_t realsize = *size;
+
+ int64_t realsize = *(int64_t*)size;
if(realsize==-1) {
DEBUG("Source CDO empty, doing NULL transfer\n");
realsize = 0;
}
- if (!g_mio_available
- || (( realsize % getpagesize()) != 0
- && init->methods->supported[0] == MSTRO__POOL__TRANSPORT_KIND__MIO)
+ if ( init->methods->supported[0] == MSTRO__POOL__TRANSPORT_KIND__MIO
+ && (!g_mio_available || (realsize % getpagesize()) != 0 )
){
WARN("Not issuing a ticket with MIO. Either not available or CDO size (%zu)"
" is not a multiple of the page size (%d)."
@@ -398,8 +433,13 @@ mstro_pc__handle_initiate_transfer(const Mstro__Pool__InitiateTransfer* init)
switch(ticket.ticket_case) {
case MSTRO__POOL__TRANSFER_TICKET__TICKET_GFS:
INFO("TICKET CASE GFS\n");
- gfs.path = src_cdo->name;
- gfs.keep_file = 0; // Arbitrarily rm the transport file on dst
+ status= mstro_pc__construct_gfs_path_for_cdo(src_cdo, &gfs.path);
+ if(status!=MSTRO_OK) {
+ ERR("Failed to construct GFS path for SRC-CDO: %d (%s)\n",
+ status, mstro_status_description(status));
+ return status;
+ }
+ gfs.keep_file = 1; // don't arbitrarily rm the transport file on dst
break;
case MSTRO__POOL__TRANSFER_TICKET__TICKET_MIO:
INFO("TICKET CASE MIO\n");
@@ -473,7 +513,7 @@ mstro_pc__handle_initiate_transfer(const Mstro__Pool__InitiateTransfer* init)
INFO("Issued ticket for CDO %s, and starting execute process\n", src_cdo->name);
//INFO("TransferTicket using path %s\n", ticket.gfs->path);
- INFO("TransferTicket cdo size %zu\n", ticket.data_size);
+ INFO("TransferTicket cdo size %" PRIi64 "\n", ticket.data_size);
/* Execute transport (non-blocking) */
status = mstro_transport_execute(src_cdo, &ticket);
diff --git a/maestro/pool_manager.c b/maestro/pool_manager.c
index 95fff9b67c9e4e78435e71ceca4a7e3e9b3a6939..5627de67fa04c3cef9b60f3f99e990fac9af2bda 100644
--- a/maestro/pool_manager.c
+++ b/maestro/pool_manager.c
@@ -441,6 +441,31 @@ mstro_pm__msg_free(Mstro__Pool__MstroMsg *msg)
mstro__pool__mstro_msg__free_unpacked(msg, NULL);
}
}
+
+
+/** if the application receiving EV needs to be told the name, send
+ * it, otherwise set it to NULL in the EV structure */
+static inline
+mstro_status
+mstro_pm__possibly_fill_event_cdoname(
+ const struct mstro_cdo_id *cdoid,
+ mstro_app_id recipient,
+ Mstro__Pool__Event *ev)
+{
+ recipient = recipient; /* avoid unused warning */
+
+ /** FIXME: for the moment we just always send the name, could be improved */
+ mstro_status s=
+ mstro_pm_cdo_registry_cdo_name_lookup(cdoid,
+ (const char **)&ev->cdo_name);
+ if(s!=MSTRO_OK) {
+ WITH_CDO_ID_STR(idstr, cdoid, {
+ ERR("Cannot find CDO registry entry for |%s|\n",
+ idstr);});
+ return MSTRO_NOENT;
+ }
+ return MSTRO_OK;
+}
/*** DECLARE ***/
@@ -505,6 +530,7 @@ mstro_pm__handle_declare_phase2(mstro_event event,
* on stack and will only be alive until notify-and-continue is
* done */
ev.origin_id = cont->msg->token->appid;
+ ev.cdo_name = cont->msg->declare->cdo_name;
status = mstro_pm__event_notify_and_continue(
&ev,
@@ -513,7 +539,7 @@ mstro_pm__handle_declare_phase2(mstro_event event,
mstro_pm__handle_declare_phase3,
cont->msg, NULL, NULL);
if(status!=MSTRO_OK) {
- ERR("Failed notifying subscribers of DECALRE:after event: %d (%s)\n",
+ ERR("Failed notifying subscribers of DECLARE:after event: %d (%s)\n",
status, mstro_status_description(status));
goto BAILOUT_FREE;
}
@@ -543,6 +569,7 @@ mstro_pm__handle_declare(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_DECLARE;
ev.declare = msg->declare;
ev.origin_id = msg->token->appid;
+ ev.cdo_name = msg->declare->cdo_name;
mstro_status status
= mstro_pm__event_notify_and_continue(
@@ -966,6 +993,10 @@ mstro_pm__handle_seal_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_SEAL;
ev.seal = cont->msg->seal;
ev.origin_id = cont->msg->token->appid;
+ mstro_status s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
status = mstro_pm__event_notify_and_continue(
&ev,
@@ -1020,6 +1051,10 @@ mstro_pm__handle_seal(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_SEAL;
ev.seal = msg->seal;
ev.origin_id = msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1088,6 +1123,10 @@ mstro_pm__handle_offer_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_OFFER;
ev.offer = cont->msg->offer;
ev.origin_id = cont->msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1114,8 +1153,10 @@ mstro_pm__handle_offer(Mstro__Pool__MstroMsg *msg)
{
Mstro__Pool__Offer *offer = msg->offer;
mstro_app_id app_id = msg->token->appid->id;
-
assert(offer!=NULL); assert(app_id!=MSTRO_APP_ID_INVALID);
+ struct mstro_cdo_id cdoid = { .qw[0] = offer->cdoid->qw0,
+ .qw[1] = offer->cdoid->qw1 };
+
DEBUG("CDO OFFER from %zu\n", app_id);
mstro_status s=MSTRO_FAIL;
@@ -1130,6 +1171,10 @@ mstro_pm__handle_offer(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_OFFER;
ev.offer = msg->offer;
ev.origin_id = msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1201,7 +1246,11 @@ mstro_pm__handle_require_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_REQUIRE;
ev.require = cont->msg->require;
ev.origin_id = cont->msg->token->appid;
-
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
+
s = mstro_pm__event_notify_and_continue(
&ev,
false,
@@ -1229,6 +1278,9 @@ mstro_pm__handle_require(Mstro__Pool__MstroMsg *msg)
Mstro__Pool__Require *require = msg->require;
mstro_app_id app_id = msg->token->appid->id;
assert(require!=NULL); assert(app_id!=MSTRO_APP_ID_INVALID);
+ struct mstro_cdo_id cdoid = { .qw[0] = require->cdoid->qw0,
+ .qw[1] = require->cdoid->qw1 };
+
DEBUG("CDO REQUIRE from %zu\n", app_id);
mstro_status s;
@@ -1243,6 +1295,10 @@ mstro_pm__handle_require(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_REQUIRE;
ev.require = msg->require;
ev.origin_id = msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1295,8 +1351,12 @@ mstro_pm__handle_retract_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_RETRACT;
ev.retract = cont->msg->retract;
ev.origin_id = cont->msg->token->appid;
+ mstro_status s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
- mstro_status s = mstro_pm__event_notify_and_continue(
+ s = mstro_pm__event_notify_and_continue(
&ev,
false,
NULL,
@@ -1322,7 +1382,13 @@ mstro_pm__handle_retract(Mstro__Pool__MstroMsg *msg)
{
Mstro__Pool__Retract *retract = msg->retract;
mstro_app_id app_id = msg->token->appid->id;
- retract = retract; app_id=app_id; /* avoid unused arg warning */
+ assert(retract!=NULL);
+
+ app_id=app_id; /* avoid unused arg warning */
+ struct mstro_cdo_id cdoid = { .qw[0] = retract->cdoid->qw0,
+ .qw[1] = retract->cdoid->qw1 };
+
+
Mstro__Pool__Event ev = MSTRO__POOL__EVENT__INIT;
/* serial and handle set later */
@@ -1330,8 +1396,13 @@ mstro_pm__handle_retract(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_RETRACT;
ev.retract = msg->retract;
ev.origin_id = msg->token->appid;
+ mstro_status s =
+ mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
- mstro_status s = mstro_pm__event_notify_and_continue(
+ s = mstro_pm__event_notify_and_continue(
&ev,
true,
NULL,
@@ -1401,6 +1472,10 @@ mstro_pm__handle_demand_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_DEMAND;
ev.demand = cont->msg->demand;
ev.origin_id = cont->msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1429,6 +1504,11 @@ mstro_pm__handle_demand(Mstro__Pool__MstroMsg *msg)
Mstro__Pool__Demand *demand = msg->demand;
mstro_app_id app_id = msg->token->appid->id;
assert(demand!=NULL); assert(app_id!=MSTRO_APP_ID_INVALID);
+
+ struct mstro_cdo_id cdoid = { .qw[0] = demand->cdoid->qw0,
+ .qw[1] = demand->cdoid->qw1 };
+
+
DEBUG("CDO DEMAND from %zu\n", app_id);
mstro_status s;
@@ -1443,6 +1523,10 @@ mstro_pm__handle_demand(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_DEMAND;
ev.demand = msg->demand;
ev.origin_id = msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1511,6 +1595,10 @@ mstro_pm__handle_withdraw_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_WITHDRAW;
ev.withdraw = cont->msg->withdraw;
ev.origin_id = cont->msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1555,6 +1643,10 @@ mstro_pm__handle_withdraw(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_WITHDRAW;
ev.withdraw = msg->withdraw;
ev.origin_id = msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1610,6 +1702,10 @@ mstro_pm__handle_dispose_phase2(mstro_event event,
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_DISPOSE;
ev.dispose = cont->msg->dispose;
ev.origin_id = cont->msg->token->appid;
+ s = mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
s = mstro_pm__event_notify_and_continue(
&ev,
@@ -1636,7 +1732,12 @@ mstro_pm__handle_dispose(Mstro__Pool__MstroMsg *msg)
{
Mstro__Pool__Dispose *dispose = msg->dispose;
mstro_app_id app_id = msg->token->appid->id;
- dispose=dispose; app_id=app_id; /* avoid unused arg warning */
+ app_id=app_id; /* avoid unused arg warning */
+ assert(dispose!=NULL);
+
+ struct mstro_cdo_id cdoid = { .qw[0] = dispose->cdoid->qw0,
+ .qw[1] = dispose->cdoid->qw1 };
+
Mstro__Pool__Event ev = MSTRO__POOL__EVENT__INIT;
/* serial and handle set later */
@@ -1644,8 +1745,13 @@ mstro_pm__handle_dispose(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_DISPOSE;
ev.dispose = msg->dispose;
ev.origin_id = msg->token->appid;
+ mstro_status s =
+ mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
- mstro_status s = mstro_pm__event_notify_and_continue(
+ s = mstro_pm__event_notify_and_continue(
&ev,
true,
NULL,
@@ -1684,6 +1790,9 @@ mstro_pm__handle_transfer_completed(Mstro__Pool__MstroMsg *msg)
mstro_app_id app_id = msg->token->appid->id;
assert(completion!=NULL); assert(app_id!=MSTRO_APP_ID_INVALID);
+ struct mstro_cdo_id cdoid = { .qw[0] = completion->cdoid->qw0,
+ .qw[1] = completion->cdoid->qw1 };
+
DEBUG("TRANSFER COMPLETION from %zu\n", app_id);
/* produce completed:after event */
@@ -1693,8 +1802,13 @@ mstro_pm__handle_transfer_completed(Mstro__Pool__MstroMsg *msg)
ev.payload_case = MSTRO__POOL__EVENT__PAYLOAD_TRANSFER_COMPLETED;
ev.transfer_completed = msg->transfer_completed;
ev.origin_id = msg->token->appid;
+ mstro_status s =
+ mstro_pm__possibly_fill_event_cdoname(&cdoid, app_id, &ev);
+ if(s!=MSTRO_OK) {
+ goto BAILOUT_FREE;
+ }
- mstro_status s = mstro_pm__event_notify_and_continue(
+ s = mstro_pm__event_notify_and_continue(
&ev,
false,
NULL,
diff --git a/maestro/pool_manager_registry.c b/maestro/pool_manager_registry.c
index bda4204003eeeeedaeb3976072700327dc49d180..af22504a0e1f9541b2bc38052a4c92df542f8685 100644
--- a/maestro/pool_manager_registry.c
+++ b/maestro/pool_manager_registry.c
@@ -1133,7 +1133,7 @@ mstro_pm_cdo_app_match(mstro_app_id origin, const struct mstro_cdo_id *id,
/* str, origin, cdo_selector, cdo_selector->query); */
/* }); */
mstro_status status = MSTRO_FAIL;
- /* now fetcch attributes from per-app-CDO entry, call subscription-module checker and return result */
+ /* now fetch attributes from per-app-CDO entry, call subscription-module checker and return result */
WITH_LOCKED_CDO_REGISTRY({
struct per_app_cdo_entry *entry;
status = mstro_pm_cdo_app_lookup(id, origin, &entry);
diff --git a/maestro/subscription_registry.c b/maestro/subscription_registry.c
index db4402a3d2912abf35f85155a9dc6fa8e5b594da..cb34610d24b8a738ae25ae3859dc5c0d310c942b 100644
--- a/maestro/subscription_registry.c
+++ b/maestro/subscription_registry.c
@@ -324,6 +324,7 @@ mstro__subscr_resolver_lookup_cdoid(const struct mstro_cdo_id *id,
goto ABORT;
}
}
+ DEBUG("Entering wait for resolver lookup reply\n");
/* setup done, time to wait. ResolveReply will be handled by
* @ref mstro_pool_resolve_reply_bh will will trigger our
* event */
@@ -1424,13 +1425,15 @@ const Mstro__Pool__KvEntry *
mstro_subscription__pool_attr_find(const Mstro__Pool__Attributes__Map *kv_map,
const char *key)
{
- if(kv_map==NULL)
+ if(kv_map==NULL) {
+ DEBUG("Empty kv map when looking for key |%s|\n", key);
return NULL;
- else {
- for(size_t i=0; i<kv_map->n_map; i++)
+ } else {
+ for(size_t i=0; i<kv_map->n_map; i++) {
if(strcmp(key,kv_map->map[i]->key)==0) {
return kv_map->map[i];
}
+ }
return NULL;
}
}
@@ -1487,18 +1490,18 @@ mstro_subscription__csq_eval(const struct mstro_csq_val *csq,
entry = mstro_subscription__pool_attr_find(attributes->kv_map, key);
if(entry==NULL) {
if(complement) {
- DEBUG("has-not matched for |%s|\n", key);
+ DEBUG("HAS-NOT matched for |%s|\n", key);
return MSTRO_OK;
} else {
- DEBUG("has did not match for |%s|\n", key);
+ DEBUG("HAS did not match for |%s|\n", key);
return MSTRO_NOMATCH;
}
} else {
if(complement) {
- DEBUG("has-not did not match for |%s|\n", key);
+ DEBUG("HAS-NOT did not match for |%s|\n", key);
return MSTRO_NOMATCH;
} else {
- DEBUG("has matched for |%s|\n", key);
+ DEBUG("HAS matched for |%s|\n", key);
return MSTRO_OK;
}
}
@@ -1526,7 +1529,8 @@ mstro_subscription__csq_eval(const struct mstro_csq_val *csq,
return MSTRO_NOMATCH;
}
} else {
- /* key matched. Now need to parse the value in the csq and call the appropriate comparison function */
+ /* key matched. Now need to parse the value in the csq and
+ * call the appropriate comparison function */
const Mstro__Pool__AVal *aval = entry->val;
void *rhsval=NULL;
size_t rhsval_size=0;
@@ -1605,7 +1609,8 @@ mstro_subscription_selector_eval(const struct mstro_cdo_id *cdoid,
struct mstro_csq_val *csq = sel->csq;
if(csq==NULL) {
- /* FIXME: maybe we need to create it on the first use of SEL if we're on the PM? */
+ /* FIXME: maybe we need to create it on the first use of SEL if
+ * we're on the PM? */
ERR("No parsed query (CSQ) available\n");
return MSTRO_FAIL;
}
@@ -1633,7 +1638,7 @@ mstro_subscription_selector_check(struct mstro_subscription_ *s,
/* declare is special: no CDOID known yet, no attribuites, only
* string name. Efficient subscriptions will subscribe to
* DECLARE_ACK instead. */
- WARN("DECLARE event always matches -- FIXME\n");
+ WARN("DECLARE event always matches -- FIXME: only name could be checked (and we don't)\n");
return MSTRO_OK;
} else {
/* we always need the CDO id. Unfortunately it's slightly buried in the EV */
@@ -1656,7 +1661,8 @@ mstro_subscription_selector_check(struct mstro_subscription_ *s,
ERR("Failed to serialized attribute message for CDO\n");
return status;
}
- status = mstro_subscription_selector_eval(&id, s->cdo_selector, attributes);
+ status = mstro_subscription_selector_eval(&id, s->cdo_selector,
+ attributes);
}
}
return status;
@@ -1756,12 +1762,15 @@ mstro_pool_event_advertise(Mstro__Pool__Event *ev,
if(s->subscription->needs_ack) {
uint64_t num_outstanding
- = atomic_fetch_add_explicit(&ctx->nr_outstanding_acks, 1, memory_order_release);
- DEBUG("Now %" PRIu64 " outstanding acks for this event\n", num_outstanding);
+ = atomic_fetch_add_explicit(&ctx->nr_outstanding_acks,
+ 1, memory_order_release);
+ DEBUG("Now %" PRIu64 " outstanding acks for this event\n",
+ num_outstanding);
mstro_event ack_event;
status = mstro_event_create(g_subscription_table.edom,
- event_ack_cb, ctx, NULL, true, &ack_event);
+ event_ack_cb, ctx, NULL, true,
+ &ack_event);
if(status!=MSTRO_OK) {
ERR("Failed to create event for subscription ack, skipping subscriber %" PRIu64 "\n",
s->subscriber);
@@ -1901,9 +1910,25 @@ mstro_pool_event_consume(const Mstro__Pool__Event *eventmsg)
const char* cdo_name;
cdo_name = malloc(MSTRO_CDO_NAME_MAX);
assert(cdo_name!=NULL);
- mstro_status s = mstro__subscr_resolver_lookup_cdoid(
- (struct mstro_cdo_id*)eventmsg->offer->cdoid, &cdo_name);
- ev->offer.cdo_name = (char*)cdo_name;
+ struct mstro_cdo_id id = {
+ .qw[0] = eventmsg->offer->cdoid->qw0,
+ .qw[1] = eventmsg->offer->cdoid->qw1
+ };
+ /* we cannot call the resolver in here, as that would lock up
+ * our thread in the PC. But CDO-related events should have a
+ * NAME value */
+ if(eventmsg->cdo_name==NULL) {
+ ERR("OFFER event missing a CDO name\n");
+ free(ev);
+ return MSTRO_FAIL;
+ } else {
+ ev->offer.cdo_name = strdup(eventmsg->cdo_name);
+ if(ev->declare.cdo_name == NULL) {
+ ERR("Failed to allocate event data\n");
+ free(ev);
+ return MSTRO_NOMEM;
+ }
+ }
DEBUG("Event: OFFER for |%s| from %" PRIu64 "\n",
ev->offer.cdo_name, ev->offer.appid);
break;
@@ -1914,23 +1939,21 @@ mstro_pool_event_consume(const Mstro__Pool__Event *eventmsg)
&& eventmsg->origin_id->id!=MSTRO_APP_ID_INVALID);
assert(eventmsg->seal->cdoid!=NULL);
ev->seal.appid = eventmsg->origin_id->id;
- {
- WARN("FIXME: Missing CDO resolver, returning ID string\n");
- struct mstro_cdo_id id = {.qw[0] = eventmsg->seal->cdoid->qw0,
- .qw[1] = eventmsg->seal->cdoid->qw1 };
-
- WITH_CDO_ID_STR(idstr, &id, {
- ev->seal.cdo_name = strdup(idstr);
- });
+ if(eventmsg->cdo_name==NULL) {
+ ERR("SEAL event missing a CDO name\n");
+ free(ev);
+ return MSTRO_FAIL;
+ } else {
+ ev->seal.cdo_name = strdup(eventmsg->cdo_name);
if(ev->seal.cdo_name == NULL) {
ERR("Failed to allocate event data\n");
free(ev);
return MSTRO_NOMEM;
}
-
- DEBUG("Event: SEAL for |%s| from %" PRIu64 "\n",
- ev->seal.cdo_name, ev->seal.appid);
}
+
+ DEBUG("Event: SEAL for |%s| from %" PRIu64 "\n",
+ ev->seal.cdo_name, ev->seal.appid);
break;
case MSTRO_POOL_EVENT_REQUIRE:
@@ -1939,23 +1962,21 @@ mstro_pool_event_consume(const Mstro__Pool__Event *eventmsg)
&& eventmsg->origin_id->id!=MSTRO_APP_ID_INVALID);
assert(eventmsg->require->cdoid!=NULL);
ev->require.appid = eventmsg->origin_id->id;
- {
- WARN("FIXME: Missing CDO resolver, returning ID string\n");
- struct mstro_cdo_id id = {.qw[0] = eventmsg->require->cdoid->qw0,
- .qw[1] = eventmsg->require->cdoid->qw1 };
-
- WITH_CDO_ID_STR(idstr, &id, {
- ev->require.cdo_name = strdup(idstr);
- });
+ if(eventmsg->cdo_name==NULL) {
+ ERR("REQUIRE event missing a CDO name\n");
+ free(ev);
+ return MSTRO_FAIL;
+ } else {
+ ev->require.cdo_name = strdup(eventmsg->cdo_name);
if(ev->require.cdo_name == NULL) {
ERR("Failed to allocate event data\n");
free(ev);
return MSTRO_NOMEM;
}
-
- DEBUG("Event: REQUIRE for |%s| from %" PRIu64 "\n",
- ev->require.cdo_name, ev->require.appid);
}
+
+ DEBUG("Event: REQUIRE for |%s| from %" PRIu64 "\n",
+ ev->require.cdo_name, ev->require.appid);
break;
case MSTRO_POOL_EVENT_WITHDRAW:
@@ -1964,23 +1985,22 @@ mstro_pool_event_consume(const Mstro__Pool__Event *eventmsg)
&& eventmsg->origin_id->id!=MSTRO_APP_ID_INVALID);
assert(eventmsg->withdraw->cdoid!=NULL);
ev->withdraw.appid = eventmsg->origin_id->id;
- {
- WARN("FIXME: Missing CDO resolver, returning ID string\n");
- struct mstro_cdo_id id = {.qw[0] = eventmsg->withdraw->cdoid->qw0,
- .qw[1] = eventmsg->withdraw->cdoid->qw1 };
-
- WITH_CDO_ID_STR(idstr, &id, {
- ev->withdraw.cdo_name = strdup(idstr);
- });
+
+ if(eventmsg->cdo_name==NULL) {
+ ERR("WITHDRAW event missing a CDO name\n");
+ free(ev);
+ return MSTRO_FAIL;
+ } else {
+ ev->withdraw.cdo_name = strdup(eventmsg->cdo_name);
if(ev->withdraw.cdo_name == NULL) {
ERR("Failed to allocate event data\n");
free(ev);
return MSTRO_NOMEM;
}
-
- DEBUG("Event: WITHDRAW for |%s| from %" PRIu64 "\n",
- ev->withdraw.cdo_name, ev->withdraw.appid);
}
+
+ DEBUG("Event: WITHDRAW for |%s| from %" PRIu64 "\n",
+ ev->withdraw.cdo_name, ev->withdraw.appid);
break;
case MSTRO_POOL_EVENT_APP_BYE:
@@ -2155,9 +2175,11 @@ mstro_subscriptions_finalize(void)
return status;
}
+/* called on pool client to handle resolver reply */
mstro_status
mstro_pool_resolve_reply_bh(const Mstro__Pool__ResolveReply *reply)
{
+ DEBUG("Resolver reply received\n");
if(reply==NULL ||reply->query==NULL) {
return MSTRO_INVMSG;
}
diff --git a/protocols/mstro_pool.pb-c.c b/protocols/mstro_pool.pb-c.c
index b08e90ef01b64e05a8f8d64d80e1fb9cd576bd34..e241057209aa3eef7544237f28ab8682b6fbc8f7 100644
--- a/protocols/mstro_pool.pb-c.c
+++ b/protocols/mstro_pool.pb-c.c
@@ -4890,7 +4890,7 @@ const ProtobufCMessageDescriptor mstro__pool__unsubscribe__descriptor =
(ProtobufCMessageInit) mstro__pool__unsubscribe__init,
NULL,NULL,NULL /* reserved[123] */
};
-static const ProtobufCFieldDescriptor mstro__pool__event__field_descriptors[23] =
+static const ProtobufCFieldDescriptor mstro__pool__event__field_descriptors[24] =
{
{
"subscription_handle",
@@ -4940,6 +4940,18 @@ static const ProtobufCFieldDescriptor mstro__pool__event__field_descriptors[23]
0, /* flags */
0,NULL,NULL /* reserved1,reserved2, etc */
},
+ {
+ "cdo_name",
+ 5,
+ PROTOBUF_C_LABEL_NONE,
+ PROTOBUF_C_TYPE_STRING,
+ 0, /* quantifier_offset */
+ offsetof(Mstro__Pool__Event, cdo_name),
+ NULL,
+ &protobuf_c_empty_string,
+ 0, /* flags */
+ 0,NULL,NULL /* reserved1,reserved2, etc */
+ },
{
"declare",
16,
@@ -5170,36 +5182,37 @@ static const ProtobufCFieldDescriptor mstro__pool__event__field_descriptors[23]
},
};
static const unsigned mstro__pool__event__field_indices_by_name[] = {
- 20, /* field[20] = bye */
- 4, /* field[4] = declare */
- 5, /* field[5] = declare_ack */
- 11, /* field[11] = demand */
- 13, /* field[13] = dispose */
- 17, /* field[17] = join */
+ 21, /* field[21] = bye */
+ 4, /* field[4] = cdo_name */
+ 5, /* field[5] = declare */
+ 6, /* field[6] = declare_ack */
+ 12, /* field[12] = demand */
+ 14, /* field[14] = dispose */
+ 18, /* field[18] = join */
2, /* field[2] = kind */
- 19, /* field[19] = leave */
- 8, /* field[8] = offer */
+ 20, /* field[20] = leave */
+ 9, /* field[9] = offer */
3, /* field[3] = origin_id */
- 9, /* field[9] = require */
- 12, /* field[12] = retract */
- 6, /* field[6] = seal */
- 7, /* field[7] = seal_group */
+ 10, /* field[10] = require */
+ 13, /* field[13] = retract */
+ 7, /* field[7] = seal */
+ 8, /* field[8] = seal_group */
1, /* field[1] = serial */
- 21, /* field[21] = subscribe */
+ 22, /* field[22] = subscribe */
0, /* field[0] = subscription_handle */
- 16, /* field[16] = transfer_completed */
- 14, /* field[14] = transport_init */
- 15, /* field[15] = transport_ticket */
- 22, /* field[22] = unsubscribe */
- 18, /* field[18] = welcome */
- 10, /* field[10] = withdraw */
+ 17, /* field[17] = transfer_completed */
+ 15, /* field[15] = transport_init */
+ 16, /* field[16] = transport_ticket */
+ 23, /* field[23] = unsubscribe */
+ 19, /* field[19] = welcome */
+ 11, /* field[11] = withdraw */
};
static const ProtobufCIntRange mstro__pool__event__number_ranges[3 + 1] =
{
{ 1, 0 },
- { 16, 4 },
- { 32, 17 },
- { 0, 23 }
+ { 16, 5 },
+ { 32, 18 },
+ { 0, 24 }
};
const ProtobufCMessageDescriptor mstro__pool__event__descriptor =
{
@@ -5209,7 +5222,7 @@ const ProtobufCMessageDescriptor mstro__pool__event__descriptor =
"Mstro__Pool__Event",
"mstro.pool",
sizeof(Mstro__Pool__Event),
- 23,
+ 24,
mstro__pool__event__field_descriptors,
mstro__pool__event__field_indices_by_name,
3, mstro__pool__event__number_ranges,
diff --git a/protocols/mstro_pool.pb-c.h b/protocols/mstro_pool.pb-c.h
index 553abf0d71b41bfc28fdab7022608418d7f3d72f..7c17cfeea05b198482f42d01174d0ccfa581c87e 100644
--- a/protocols/mstro_pool.pb-c.h
+++ b/protocols/mstro_pool.pb-c.h
@@ -1143,7 +1143,8 @@ struct _Mstro__Pool__Event
{
ProtobufCMessage base;
/*
- ** the (workflow-unique) event id. IDs can be reused over the course of a workflow.
+ ** the (workflow-unique) event id.
+ *IDs can be reused over the course of a workflow.
*/
Mstro__Pool__SubscriptionHandle *subscription_handle;
uint64_t serial;
@@ -1152,9 +1153,14 @@ struct _Mstro__Pool__Event
*/
Mstro__Pool__EventKind kind;
/*
- ** the application that caused the event. Can be invalid or pool manager, or a joined component
+ ** the application that caused the event.
+ *Can be invalid or pool manager, or a joined component
*/
Mstro__Pool__Appid *origin_id;
+ /*
+ ** the CDO concerned. Will be unset for non-CDO events
+ */
+ char *cdo_name;
Mstro__Pool__Event__PayloadCase payload_case;
union {
/*
@@ -1240,7 +1246,7 @@ struct _Mstro__Pool__Event
};
#define MSTRO__POOL__EVENT__INIT \
{ PROTOBUF_C_MESSAGE_INIT (&mstro__pool__event__descriptor) \
- , NULL, 0, MSTRO__POOL__EVENT_KIND__INVALID_EVENT, NULL, MSTRO__POOL__EVENT__PAYLOAD__NOT_SET, {0} }
+ , NULL, 0, MSTRO__POOL__EVENT_KIND__INVALID_EVENT, NULL, (char *)protobuf_c_empty_string, MSTRO__POOL__EVENT__PAYLOAD__NOT_SET, {0} }
struct _Mstro__Pool__EventAck
diff --git a/protocols/mstro_pool.proto b/protocols/mstro_pool.proto
index d0bc27f0ee865de7d0053b4c055e04f742c0378c..8cc097b041ebc218bb4f6845b31b652a5b750bb8 100644
--- a/protocols/mstro_pool.proto
+++ b/protocols/mstro_pool.proto
@@ -660,17 +660,22 @@ message Unsubscribe {
/** Events that match a subscription */
message Event {
- /** the (workflow-unique) event id. IDs can be reused over the course of a workflow. */
+ /** the (workflow-unique) event id.
+ IDs can be reused over the course of a workflow. */
SubscriptionHandle subscription_handle = 1;
fixed64 serial = 2;
/** The kind of event */
EventKind kind = 3;
- /** the application that caused the event. Can be invalid or pool manager, or a joined component */
+ /** the application that caused the event.
+ Can be invalid or pool manager, or a joined component */
Appid origin_id = 4;
+ /** the CDO concerned. Will be unset for non-CDO events */
+ string cdo_name = 5;
/** the data associated with the event
*
* In most cases this conincides with the payload sent int the
- * original message associated with the event, except that VSM messages are unsupported.
+ * original message associated with the event,
+ * except that VSM messages are unsupported.
*/
oneof payload {
/** for kind DECLARE */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 8beb8d0608c58da39e00a75edea32c8a8f3700df..e9c51d30a780227fa2412cc8763e3f7702d661a2 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -76,16 +76,16 @@ TESTS = check_version check_init check_uuid \
check_pm_declare.sh \
check_pm_interlock.sh \
check_subscribe.sh \
- check_pm_declare_group.sh
+ check_pm_declare_group.sh
XFAIL_TESTS = \
- check_subscribe_local
+ check_subscribe_local
# too expensive, actually more of a benchmark:
-# check_pool_local_multi
+# check_pool_local_multi
# broken for now:
-# check_mempool
+# check_mempool
check_PROGRAMS = check_version check_init check_uuid \
coverage check_memlock \
@@ -117,13 +117,13 @@ check_PROGRAMS = check_version check_init check_uuid \
simple_injector \
simple_archiver \
simple_telemetry_listener \
- check_events
+ check_events
if WITH_MIO
AM_CPPFLAGS += $(TEST_INCLUDES) -I$(top_srcdir)/deps/mio/src -D_REENTRANT -D_GNU_SOURCE -DM0_INTERNAL= -DM0_EXTERN=extern -Wno-attributes
check_PROGRAMS += check_transport_mio
-TESTS += check_transport_mio
+TESTS += check_transport_mio
endif
CLIENT1_OPS="DECLARE cdo1 1025\
@@ -155,9 +155,9 @@ INJECTOR_OPS="\
-simple_interlock_client_1_SOURCES = simple_interlock_client.c
+simple_interlock_client_1_SOURCES = simple_interlock_client.c
simple_interlock_client_1_CPPFLAGS = $(AM_CPPFLAGS) -DCLIENT_ARGS=$(CLIENT1_OPS)
-simple_interlock_client_2_SOURCES = simple_interlock_client.c
+simple_interlock_client_2_SOURCES = simple_interlock_client.c
simple_interlock_client_2_CPPFLAGS = $(AM_CPPFLAGS) -DCLIENT_ARGS=$(CLIENT2_OPS)
simple_injector_SOURCES = simple_interlock_client.c
simple_injector_CPPFLAGS = $(AM_CPPFLAGS) -DCLIENT_ARGS=$(INJECTOR_OPS)
@@ -165,14 +165,16 @@ simple_injector_CPPFLAGS = $(AM_CPPFLAGS) -DCLIENT_ARGS=$(INJECTOR_OPS)
simple_group_injector_SOURCES = simple_group_client.c
simple_group_injector_CPPFLAGS = $(AM_CPPFLAGS) -DINJECT_GROUP_MEMBERS=1
-#check_mempool
+#check_mempool
check_SCRIPTS = run_demo.sh \
check_pm_declare.sh \
check_pm_declare_group.sh \
check_pm_interlock.sh \
check_subscribe.sh
-EXTRA_DIST = $(check_SCRIPTS) generate-mio-config.sh demo_mvp_d3_2_config.yaml run_demo.sh
+user_defined_test_schemas = benchmark_attributes.yaml test_attributes.yaml
+
+EXTRA_DIST = $(check_SCRIPTS) $(user_defined_test_schemas) generate-mio-config.sh demo_mvp_d3_2_config.yaml run_demo.sh
clean-local: clean-local-check
.PHONY: clean-local-check
@@ -181,6 +183,11 @@ clean-local-check:
-rm -f $(top_builddir)/tests/consumer_CDO\ [0-9]*
-rm -f $(top_builddir)/tests/mio-config*.yaml
-rm -f $(top_builddir)/tests/m0trace.*
+ -rm -f $(top_builddir)/tests/C-handle\ [0-9]*
+ -rm -f $(top_builddir)/tests/C-name\ [0-9]*
+ -rm -f $(top_builddir)/tests/{Consumer,Producer}-done-CDO
+ -rm -f $(top_builddir)/tests/Test\ group
+ -rm -f $(top_builddir)/tests/cdo[0-9]*
.PHONY: all
diff --git a/tests/benchmark_attributes.yaml b/tests/benchmark_attributes.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4d2f49cc540b9a768aeae991c8731af542d61476
--- /dev/null
+++ b/tests/benchmark_attributes.yaml
@@ -0,0 +1,30 @@
+# A user-defined schema. The minimum is name and version
+schema-name: Benchmark Attributes
+schema-version: 0
+
+schema-namespace: ".maestro.benchmark."
+
+# attributes section is optional; if given it needs to have a sequence value
+maestro-attributes:
+
+# Top-level attributes
+
+ - key: "b_1"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "b_2"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+ - key: "b_3"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
+
+
\ No newline at end of file
diff --git a/tests/check_cdo_selectors.c b/tests/check_cdo_selectors.c
index e2ed5bfb75127893f7ce435c2026fef5ba8ddab2..6a55f3c22992cea584e74fc88925cf90f42446a8 100644
--- a/tests/check_cdo_selectors.c
+++ b/tests/check_cdo_selectors.c
@@ -154,4 +154,12 @@ CHEAT_TEST(core_cdo_selector_or_and,
)
-
+CHEAT_TEST(ecmwf_int_comparison,
+ /* https://gitlab.version.fz-juelich.de/maestro/maestro-core/-/issues/16 */
+ /* parsing of integral CDO selector values */
+ const char *s = "(.maestro.ecmwf.param = 2)";
+ cheat_assert(MSTRO_OK==mstro_selector_parse(s,&q));
+ cheat_yield();
+ cheat_assert(MSTRO_OK==mstro_csq_val_dispose(q));
+ cheat_yield();
+ )
diff --git a/tests/check_declaration_seal.c b/tests/check_declaration_seal.c
index b00a215d86f0b1f8a0b50106e23ac64930ad8796..a7266c6c0da0f5a1aa04251d248fb7e5c0b75212 100644
--- a/tests/check_declaration_seal.c
+++ b/tests/check_declaration_seal.c
@@ -39,6 +39,15 @@
#define __BASE_FILE__ __FILE__
#endif
+#ifdef TOPSRCDIR
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+
+#define SRC_PATH TOSTRING(TOPSRCDIR) "/tests"
+#else
+#define SRC_PATH "."
+#endif
+
#include "cheat.h"
#include "maestro.h"
@@ -52,7 +61,7 @@ CHEAT_TEST(cdo_declaration_seal_works,
mstro_cdo cdo=NULL;
enum mstro_cdo_attr_value_type type;
const void* val;
- uint64_t size;
+ int64_t size;
size = 16000;
cheat_assert(MSTRO_OK
== mstro_cdo_declare(
@@ -88,16 +97,16 @@ CHEAT_TEST(cdo_declaration_seal_works,
cheat_assert(!(MSTRO_OK
== mstro_cdo_attribute_set(
cdo, "maestro.core.cdo.scope.local-size",
- &size))); // not an absolute path, not a valid relative path
+ &size, true))); // not an absolute path, not a valid relative path
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_set(
cdo, ".maestro.core.cdo.scope.local-size",
- &size));
+ &size, true));
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo));
-
+
cheat_assert(MSTRO_FAIL
== mstro_cdo_attribute_set_yaml(
cdo, "allocate-now: \"no\""));
@@ -106,26 +115,26 @@ CHEAT_TEST(cdo_declaration_seal_works,
cdo, "allocate-now", &type, &val));
fprintf(stdout, "check: allocate-now: %d (%p)\n", *(const bool*)val, val);
-
+
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_get(
- cdo, "persist", &type, &val));
+ cdo, "persist", &type, &val));
fprintf(stdout, "check: persist: %d (%p)\n", *(const bool*)val, val);
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_get(
- cdo, "name", &type, &val));
+ cdo, "name", &type, &val));
fprintf(stdout, "check: name == %s (%p)\n", (const char*)val, val);
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_get(
- cdo, "scope.local-size", &type, &val));
+ cdo, "scope.local-size", &type, &val));
fprintf(stderr, "check: scope.local-size == %d (%p)\n",
*(const int*)val, val);
cheat_assert(MSTRO_NOENT
== mstro_cdo_attribute_get(
- cdo, "totally_an_attribute", &type, &val));
+ cdo, "totally_an_attribute", &type, &val));
cheat_assert(MSTRO_OK == mstro_cdo_dispose(cdo));
@@ -133,3 +142,53 @@ CHEAT_TEST(cdo_declaration_seal_works,
)
+/* put a path to user-defined schema*/
+
+CHEAT_TEST(user_defined_schemas_works,
+
+ setenv("MSTRO_SCHEMA_LIST","test_attributes.yaml", 1);
+ setenv("MSTRO_SCHEMA_PATH",SRC_PATH, 1);
+
+ cheat_assert(MSTRO_OK == mstro_init("Tests","DECLARE",0));
+
+ mstro_cdo cdo;
+ cheat_assert(MSTRO_OK ==mstro_cdo_declare("test cdo", MSTRO_ATTR_DEFAULT, &cdo));
+ cheat_assert(MSTRO_OK ==mstro_cdo_attribute_set(cdo, ".maestro.test.t_1", "test value",true));
+ cheat_assert(MSTRO_OK ==mstro_cdo_declaration_seal(cdo));
+
+ cheat_assert(MSTRO_OK ==mstro_cdo_offer(cdo));
+
+ cheat_assert(MSTRO_OK ==mstro_cdo_withdraw(cdo));
+ cheat_assert(MSTRO_OK ==mstro_cdo_dispose(cdo));
+ cheat_assert(MSTRO_OK == mstro_finalize());
+
+ )
+
+/* put a path to user-defined schema with wrong path and wrong separators*/
+
+CHEAT_TEST(user_defined_schemas_doesnot_exist,
+
+ setenv("MSTRO_SCHEMA_LIST", "user_attributes.yaml", 1);
+ cheat_assert(MSTRO_OK != mstro_init("Tests","DECLARE",0));
+ cheat_assert(MSTRO_FAIL == mstro_finalize());
+
+ )
+
+CHEAT_TEST(user_defined_schemas_paths,
+ setenv("MSTRO_SCHEMA_LIST","benchmark_attributes.yaml;test_attributes.yaml",1);
+ setenv("MSTRO_SCHEMA_PATH",SRC_PATH, 1);
+
+ cheat_assert(MSTRO_OK == mstro_init("Tests","DECLARE",0));
+ mstro_cdo cdo;
+ cheat_assert(MSTRO_OK ==mstro_cdo_declare("test cdo", MSTRO_ATTR_DEFAULT, &cdo));
+ cheat_assert(MSTRO_OK ==mstro_cdo_attribute_set(cdo, ".maestro.test.t_1", "test value", true));
+ cheat_assert(MSTRO_OK ==mstro_cdo_attribute_set(cdo, ".maestro.benchmark.b_1", "bechmark value", true));
+ cheat_assert(MSTRO_OK ==mstro_cdo_declaration_seal(cdo));
+
+ cheat_assert(MSTRO_OK ==mstro_cdo_offer(cdo));
+
+ cheat_assert(MSTRO_OK ==mstro_cdo_withdraw(cdo));
+ cheat_assert(MSTRO_OK ==mstro_cdo_dispose(cdo));
+ cheat_assert(MSTRO_OK == mstro_finalize());
+
+ )
diff --git a/tests/check_layout.c b/tests/check_layout.c
index e3d4f4190fefa72a2b88e4415d8a81ebf60d2033..3570ff2ac0c3ac7957d2ef37b17d780902f822fa 100644
--- a/tests/check_layout.c
+++ b/tests/check_layout.c
@@ -130,32 +130,32 @@ CHEAT_TEST(layout_attribute_works,
cheat_assert(MSTRO_OK == mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, &cdo_src));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- src_data));
+ src_data, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- (void**)&bytes));
+ (void**)&bytes, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
- MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
- MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_src));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_src, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_src));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_src, true));
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo_src));
cheat_assert(MSTRO_OK == mstro_cdo_offer(cdo_src));
/* Consumer side */
cheat_assert(MSTRO_OK == mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, &cdo_dst));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_dst,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_dst,
- MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_dst,
- MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_dst));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_dst, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_dst,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_dst));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_dst, true));
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo_dst));
cheat_assert(MSTRO_OK == mstro_cdo_require(cdo_dst));
@@ -167,18 +167,18 @@ CHEAT_TEST(layout_attribute_works,
cheat_assert(MSTRO_OK == mstro_cdo_declare(name_unpooled, MSTRO_ATTR_DEFAULT, &cdo_unpooled));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled,
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- (void**)&bytes));
+ (void**)&bytes, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- unpooled_data));
+ unpooled_data, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled,
- MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled,
- MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_dst));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_dst, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_dst));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_dst, true));
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo_unpooled));
/* at this point cdo_unpooled has a mamba array (raw-ptr wrapper) */
@@ -200,18 +200,18 @@ CHEAT_TEST(layout_attribute_works,
cheat_assert(MSTRO_OK == mstro_cdo_declare(name_unpooled, MSTRO_ATTR_DEFAULT, &cdo_unpooled_t));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled_t,
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- (void**)&bytes));
+ (void**)&bytes, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled_t,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- unpooled_data_t));
+ unpooled_data_t, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled_t,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ELEMENT_SIZE, &elsz, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled_t,
- MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_NDIMS, &ndims, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled_t,
- MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_src));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_DIMS_SIZE, dimsz_src, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_unpooled_t,
- MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_src));
+ MSTRO_ATTR_CORE_CDO_LAYOUT_ORDER, &patt_src, true));
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo_unpooled_t));
cheat_assert(MSTRO_OK == mstro_transform_layout(cdo_unpooled, cdo_unpooled_t,
diff --git a/tests/check_pool_local_rawptr.c b/tests/check_pool_local_rawptr.c
index 24ff530aefd8ebecff4fe83465599e23c7d69381..55ba01edf81f3bbf7444eab08b5b507890222438 100644
--- a/tests/check_pool_local_rawptr.c
+++ b/tests/check_pool_local_rawptr.c
@@ -77,10 +77,10 @@ CHEAT_TEST(cdo_local_pool_works,
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(src_cdo,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- src_data));
+ src_data, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(dst_cdo,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- dst_data));
+ dst_data, false));
cheat_yield();
cheat_assert(MSTRO_OK == mstro_cdo_require(dst_cdo));
cheat_yield();
diff --git a/tests/check_pool_mamba.c b/tests/check_pool_mamba.c
index d543c6d4fa7be15d08316e46f5bc5a5132935dd8..b7f9ec4c0e1ea630cd83fdc8cbe685fa334be572 100644
--- a/tests/check_pool_mamba.c
+++ b/tests/check_pool_mamba.c
@@ -1,6 +1,6 @@
/* -*- mode:c -*- */
/** @file
- ** @brief simple mamba use through maestro check
+ ** @brief simple mamba use through maestro check
**/
/*
@@ -50,31 +50,31 @@ CHEAT_TEST(cdo_local_pool_mamba_works,
char name[] = "Test CDO name üÜöÖäÄ";
mstro_cdo cdo=NULL;
cheat_assert(MSTRO_OK == mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, &cdo));
- uint64_t size = 4096;
+ int64_t size = 4096;
enum mstro_cdo_attr_value_type type;
float* buf;
void *rec;
buf = malloc(sizeof(float)*size);
fprintf(stderr, "buf: %p\n", buf);
rec = NULL;
-
+
// TODO declare/set a Mamba array
- mmbArray* ma_ptr = NULL;
+ mmbArray* ma_ptr = NULL;
// TODO associate (cdo, mmb_array_ptr)
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_set(
- cdo, ".maestro.core.cdo.raw-ptr", (void*)buf));
+ cdo, ".maestro.core.cdo.raw-ptr", (void*)buf, false));
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_set(
- cdo, ".maestro.core.cdo.scope.local-size", &size));
-
+ cdo, ".maestro.core.cdo.scope.local-size", &size, true));
+
mmbArray *mamba_ptr=NULL;
/* mamba array not automatically available */
mstro_status s1 = mstro_cdo_attribute_get(
cdo, ".maestro.core.cdo.mamba-array", &type, (const void**)&mamba_ptr);
-
+
fprintf(stderr,"Mamba array fetch stat 1: %d, val %p\n", s1, mamba_ptr);
-
+
cheat_assert(MSTRO_OK==s1);
cheat_assert(mamba_ptr==NULL);
@@ -82,10 +82,10 @@ CHEAT_TEST(cdo_local_pool_mamba_works,
mmbArray *tmp=NULL;
cheat_assert(MSTRO_OK==mstro_cdo_access_mamba_array(cdo,&tmp));
cheat_assert(tmp==NULL);
-
+
/* now seal */
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo));
-
+
cheat_assert(MSTRO_OK
== mstro_cdo_access_ptr(
@@ -93,7 +93,7 @@ CHEAT_TEST(cdo_local_pool_mamba_works,
fprintf(stderr, "rec: %p\n", (float*)rec);
cheat_assert(rec==buf);
- /* afterwards we can also get mamba array as an attribute */
+ /* afterwards we can also get mamba array as an attribute */
cheat_assert(MSTRO_OK
== mstro_cdo_attribute_get(
cdo, ".maestro.core.cdo.mamba-array", &type, (const void**)&mamba_ptr));
@@ -114,6 +114,3 @@ CHEAT_TEST(cdo_local_pool_mamba_works,
cheat_assert(MSTRO_OK == mstro_finalize());
free(buf);
)
-
-
-
diff --git a/tests/check_transport_gfs.c b/tests/check_transport_gfs.c
index 749a03415b7de2c3596240b5b1c03850fe4875e7..05e651307c9f482b65c9d59ec8f5840ce54595d5 100644
--- a/tests/check_transport_gfs.c
+++ b/tests/check_transport_gfs.c
@@ -87,10 +87,10 @@ CHEAT_TEST(transport_gfs_works,
cheat_assert(MSTRO_OK == mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, &cdo_src));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- src_data));
+ src_data, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- (void**)&bytes));
+ (void**)&bytes, true));
cheat_yield();
enum mstro_cdo_attr_value_type ttype;
@@ -141,7 +141,7 @@ CHEAT_TEST(transport_gfs_works,
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_dst,
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- (void**)&bytes));
+ (void**)&bytes, true));
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo_dst));
cheat_assert(MSTRO_OK == mstro_transport_gfs_dst_execute(cdo_dst, &ticket));
diff --git a/tests/check_transport_mio.c b/tests/check_transport_mio.c
index a2e8a0a187da9cc750ceb891bab4bee0bddbb3f7..fd7116c8034e742b92be20b87a7f03cf0919879d 100644
--- a/tests/check_transport_mio.c
+++ b/tests/check_transport_mio.c
@@ -58,26 +58,26 @@ CHEAT_DECLARE (
{
unsigned char x;
int i;
-
- for (i=0,x=0;i<size; i++)
- x ^= ((unsigned char*)rawptr)[i];
-
+
+ for (i=0,x=0;i<size; i++)
+ x ^= ((unsigned char*)rawptr)[i];
+
fprintf(stderr, "Checksum for cdo \"%s\":\t%d\n", name, x);
return x;
}
)
-
+
/* this tests MIO transport by using this single thread as both Producer and Consumer */
CHEAT_TEST(transport_mio_works,
size_t data_count = 1031;
- size_t bytes = data_count*sizeof(double);
+ int64_t bytes = data_count*sizeof(double);
size_t pad = 0;
double src_data[data_count];
for(size_t i=0; i<data_count; i++) {
src_data[i]=random();
}
unsigned char sender_checksum = transport_checksum("pioneer departure", src_data, data_count);
-
+
cheat_assert(MSTRO_OK == mstro_init("Tests","TRANSPORT",0));
char name[] = "transport_pioneer";
char filename_dst[] = "./CDOs/__CDO_mio_transport_pioneer";
@@ -85,17 +85,17 @@ CHEAT_DECLARE (
mstro_cdo cdo_dst=NULL;
int s = mkdir("./CDOs", S_IRWXU);
cheat_assert(s==0 || errno==EEXIST);
-
+
cheat_assert(MSTRO_OK == mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, &cdo_src));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- src_data));
+ src_data, false));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
".maestro.core.cdo.scope.local-size",
- &bytes));
+ &bytes, true));
cheat_assert(MSTRO_OK == mstro_cdo_attribute_set(cdo_src,
".maestro.core.cdo.layout.pre-pad",
- &pad));
+ &pad, true));
const int64_t* tval;
@@ -110,13 +110,13 @@ CHEAT_DECLARE (
NULL,
(const void**)&tval));
cheat_assert(*tval==bytes);
-
+
cheat_assert(MSTRO_OK == mstro_cdo_declare(name, MSTRO_ATTR_DEFAULT, &cdo_dst));
-
+
/* Particular case where the offerer is also the src, so let us execute
* the src transport before offering, otherwise the raw-ptr won't be
* accessible anymore */
-
+
/* Have a ticket issued for src */
Mstro__Pool__UUID id = MSTRO__POOL__UUID__INIT;
size_t i;
@@ -147,10 +147,10 @@ CHEAT_DECLARE (
ticket.ticket_case = MSTRO__POOL__TRANSFER_TICKET__TICKET_MIO;
Mstro__Pool__TransferTicketMIO mio = MSTRO__POOL__TRANSFER_TICKET_MIO__INIT;
ticket.mio = &mio;
- struct mstro_cdo_id* semid;
+ struct mstro_cdo_id* semid;
semid = malloc(sizeof(struct mio_obj_id));
cheat_assert(semid != NULL);
- struct mstro_cdo_id* objid;
+ struct mstro_cdo_id* objid;
objid = malloc(sizeof(struct mio_obj_id));
cheat_assert (objid != NULL);
char* semname = NULL;
@@ -170,53 +170,50 @@ CHEAT_DECLARE (
*/
cheat_assert(sizeof(struct mstro_cdo_id) == 2*sizeof(uint64_t));
cheat_assert(sizeof(struct mstro_cdo_id) == sizeof(struct mio_obj_id));
-
+
ticket.mio->semid.len = sizeof(struct mstro_cdo_id);
ticket.mio->semid.data = (uint8_t*)semid;
ticket.mio->objid.len = sizeof(struct mstro_cdo_id);
objid->qw[0] = ticket.cdoid->qw0;
objid->qw[1] = ticket.cdoid->qw1;
ticket.mio->objid.data = (uint8_t*)objid;
-
+
ticket.mio->keep_obj = 0;
/* No clean issuing, have to duplicate code above from maestro/ofi.c*/
// cheat_assert(MSTRO_OK == mstro_transport_ticket_issue(cdo_src, &ticket));
-
- cheat_assert(MSTRO_OK == mstro_transport_mio_src_execute(cdo_src, &ticket));
-
+
+ cheat_assert(MSTRO_OK == mstro_transport_mio_src_execute(cdo_src, &ticket));
+
cheat_assert(MSTRO_OK == mstro_cdo_offer(cdo_src));
/* ticket is supposed to be sent to dst (4-step protocol), so here let
* us just pretend it was, and use ticket directly for dst transport
* execute, while it is being implemented */
-
+
cheat_assert(MSTRO_OK == mstro_cdo_declaration_seal(cdo_dst));
- cheat_assert(MSTRO_OK == mstro_transport_mio_dst_execute(cdo_dst, &ticket));
+ cheat_assert(MSTRO_OK == mstro_transport_mio_dst_execute(cdo_dst, &ticket));
cheat_assert(MSTRO_OK == mstro_cdo_require(cdo_dst));
cheat_assert(MSTRO_OK == mstro_cdo_demand(cdo_dst));
-
-
+
+
double* data;
size_t len;
enum mstro_cdo_attr_value_type type;
const void* val;
-
+
cheat_assert(MSTRO_OK == mstro_cdo_access_ptr(cdo_dst, (void**)&data, NULL));
- cheat_assert(MSTRO_OK == mstro_cdo_attribute_get(cdo_dst, "scope.local-size", &type, &val));
+ cheat_assert(MSTRO_OK == mstro_cdo_attribute_get(cdo_dst, "scope.local-size", &type, &val));
len = *(size_t*)val;
len /= sizeof(double);
cheat_assert(len > 0 && data != NULL);
unsigned char receiver_checksum = transport_checksum("pioneer arrival", data, len);
cheat_assert(sender_checksum == receiver_checksum);
-
+
cheat_assert(MSTRO_OK == mstro_cdo_dispose(cdo_dst));
cheat_assert(MSTRO_OK == mstro_cdo_withdraw(cdo_src));
cheat_assert(MSTRO_OK == mstro_cdo_dispose(cdo_src));
cheat_assert(MSTRO_OK == mstro_finalize());
-
- )
-
-
+ )
diff --git a/tests/demo_mvp_d3_2.c b/tests/demo_mvp_d3_2.c
index 95082a69e222054b950c07c59874a1aed003c9c8..0855944df6759324aa0314465b09192a4558d073 100644
--- a/tests/demo_mvp_d3_2.c
+++ b/tests/demo_mvp_d3_2.c
@@ -208,7 +208,7 @@ static const cyaml_config_t config = {
int
mvp_config_parse(mvp_config* handle, const char* filename)
{
- if (handle == NULL) return MSTRO_INVARG;
+ if (handle == NULL) return MSTRO_INVARG;
cyaml_err_t err;
err = cyaml_load_file(filename, &config,
@@ -237,7 +237,7 @@ wait_for_announcement(struct cdo_announcement *announcement)
}
s = mstro_cdo_attribute_set(announcement_cdo,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- announcement);
+ announcement, false);
if(s!=MSTRO_OK) {
ERR("Failed to set raw-ptr attribute on announcement CDO\n");
abort();
@@ -255,7 +255,7 @@ wait_for_announcement(struct cdo_announcement *announcement)
ERR("Failed to require announcement CDO\n");
abort();
}
-
+
s = mstro_cdo_demand(announcement_cdo);
if(s!=MSTRO_OK) {
ERR("Failed to withdraw announcement CDO\n");
@@ -286,7 +286,7 @@ do_announce(struct cdo_announcement *announcement, mstro_cdo *result)
}
s = mstro_cdo_attribute_set(announcement_cdo,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- announcement);
+ announcement, false);
if(s!=MSTRO_OK) {
ERR("Failed to set raw-ptr attribute on announcement CDO\n");
@@ -450,7 +450,7 @@ archiver_flush_to_disk(const char *name, mmbArray *a)
abort();
}
- size_t nt;
+ size_t nt;
stat = mmb_tile_iterator_count(it, &nt);
if(stat != MMB_OK) {
ERR("Failed to get tile iterator count\n");
@@ -550,10 +550,10 @@ archiver_thread_fun(void *closure)
}
s = mstro_cdo_attribute_set(incoming[i],
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- incoming_buffers[i]);
+ incoming_buffers[i], false);
s |= mstro_cdo_attribute_set(incoming[i],
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- &announcement->cdo_size);
+ &announcement->cdo_size, true);
// INFO("archiver cdo %d incoming buffer %p\n", i, incoming_buffers[i]);
if(s!=MSTRO_OK) {
@@ -671,10 +671,10 @@ producer_thread_fun(void *closure)
s = mstro_cdo_attribute_set(outgoing[i],
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- outgoing_buffers[i]);
+ outgoing_buffers[i], false);
s |= mstro_cdo_attribute_set(outgoing[i],
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- &announcement->cdo_size);
+ &announcement->cdo_size, true);
if(s!=MSTRO_OK) {
ERR("Failed to add outgoing buffer to CDO %s\n",
@@ -721,8 +721,8 @@ mvp_checksum(const char* name, void* rawptr, uint64_t size)
unsigned char x;
uint64_t i;
- for (i=0,x=0;i<size; i++)
- x ^= ((unsigned char*)rawptr)[i];
+ for (i=0,x=0;i<size; i++)
+ x ^= ((unsigned char*)rawptr)[i];
INFO("Checksum for cdo \"%s\": %d\n", name, x);
}
@@ -731,7 +731,7 @@ void
consumer_flush_to_disk(const char *name, void *a, uint64_t size)
{
char file_name [256];
- sprintf(file_name, "consumer_%s", name);
+ sprintf(file_name, "consumer_%s", name);
FILE *dst = fopen((const char*)file_name, "w");
if(dst==NULL) {
@@ -769,7 +769,7 @@ consumer_thread_fun(void *closure)
mstro_status s;
size_t my_idx = * (size_t*)closure;
INFO("Consumer %zu starting\n", my_idx);
-
+
struct cdo_announcement *announcement
= malloc(sizeof(struct cdo_announcement));
if(announcement==NULL) {
@@ -803,10 +803,10 @@ consumer_thread_fun(void *closure)
}
s = mstro_cdo_attribute_set(incoming[i],
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- incoming_buffers[i]);
+ incoming_buffers[i], false);
s |= mstro_cdo_attribute_set(incoming[i],
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- &announcement->cdo_size);
+ &announcement->cdo_size, true);
// INFO("consumer cdo %d incoming buffer %p\n", i, incoming_buffers[i]);
if(s!=MSTRO_OK) {
@@ -826,7 +826,7 @@ consumer_thread_fun(void *closure)
/* send ACK that we're ready */
mstro_cdo ready_cdo;
-
+
INFO("Declaring Consumer %zu ready\n", my_idx);
declare_consumer_ready(my_idx,&ready_cdo);
@@ -853,14 +853,14 @@ consumer_thread_fun(void *closure)
/* query the size */
s = mstro_cdo_attribute_get(incoming[i],
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- &type, (const void**)&size);
+ &type, (const void**)&size);
if(s!=MSTRO_OK) {
ERR("Failed to extract local size from CDO %s for archiving\n",
announcement->cdo_names[i]);
abort();
}
- /* do some work */
+ /* do some work */
mvp_checksum(announcement->cdo_names[i], rawptr, *size);
/* write-out */
@@ -878,7 +878,7 @@ consumer_thread_fun(void *closure)
}
/* demand data and process */
-
+
cdo_sem_post_cleanup(ready_cdo);
free(announcement);
@@ -893,7 +893,15 @@ consumer_thread_fun(void *closure)
#define DEFAULT_CDO_SIZE 1024
#define DEFAULT_CDO_COUNT 42
+#ifdef TOPSRCDIR
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+
+#define CONFIG_FILE_PATH TOSTRING(TOPSRCDIR) "/tests/demo_mvp_d3_2_config.yaml"
+#else
#define CONFIG_FILE_PATH "./demo_mvp_d3_2_config.yaml"
+#endif
+
/*
* main thread:
@@ -933,7 +941,7 @@ int main(int argc, char **argv)
WARN("Command line arguments ignored\n");
/* fallthrough: */
case 1:
-
+
err = mvp_config_parse(&config_handle, CONFIG_FILE_PATH);
if (err != 0) {
WARN("Failed to parse a config file. Using default config.\n");
@@ -949,12 +957,12 @@ int main(int argc, char **argv)
announcement.num_archivers = config_handle->num_archivers;
announcement.cdo_size = config_handle->cdo_size;
announcement.num_entries = config_handle->cdo_count;
- }
+ }
break;
}
assert(announcement.num_entries<=CDO_COUNT_MAX);
-
+
pthread_t producers[announcement.num_producers];
pthread_t consumers[announcement.num_consumers];
pthread_t archivers[announcement.num_archivers];
@@ -1035,7 +1043,7 @@ int main(int argc, char **argv)
for(size_t i=0; i<announcement.num_archivers; i++) {
pthread_join(archivers[i], NULL);
}
-
+
/* withdraw announcement */
withdraw_announcement(announcement_cdo);
@@ -1050,11 +1058,11 @@ int main(int argc, char **argv)
announcement.num_producers,
announcement.num_consumers,
announcement.num_archivers);
-
+
BAILOUT:
if(config_handle) free(config_handle);
-
+
return s;
}
diff --git a/tests/simple_archiver.c b/tests/simple_archiver.c
index 1bf74c66d8330495b876358186c10691e265f87f..85c24ada8e1355ca775da8c0094511f50de9afa8 100644
--- a/tests/simple_archiver.c
+++ b/tests/simple_archiver.c
@@ -144,24 +144,43 @@ CHEAT_TEST(simple_archiver,
mstro_pool_event tmp=e;
/* handle all */
while(tmp) {
+ const char *event_name=NULL;
+ const char *cdo_name=NULL;
+ event_name = mstro_pool_event_description(tmp->kind);
+
switch(tmp->kind) {
case MSTRO_POOL_EVENT_OFFER:
/* FIXME: Immediately post a REQUIRE for it */
- fprintf(stderr, "Spotted an OFFER for CDO (`%s`)\n", e->offer.cdo_name);
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_DECLARE:
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_DISPOSE:
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_SEAL:
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_DEMAND:
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_REQUIRE:
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_RETRACT:
+ cdo_name = tmp->offer.cdo_name;
+ break;
case MSTRO_POOL_EVENT_WITHDRAW:
- fprintf(stdout, "CDO event %s\n",
- mstro_pool_event_description(tmp->kind));
+ cdo_name = tmp->offer.cdo_name;
break;
default:
fprintf(stderr, "Unexpected CDO event %d\n", tmp->kind);
}
+ fprintf(stdout, "CDO event %s for CDO |%s|\n",
+ event_name, cdo_name ? cdo_name : "??");
+
tmp=tmp->next;
}
/* acknowledge all */
diff --git a/tests/simple_interlock_client.c b/tests/simple_interlock_client.c
index c71f47a64a6b97c969e1795fb7a923d7aae7ac03..5b4e4fde9a66581505adde34535c116bd54f8e95 100644
--- a/tests/simple_interlock_client.c
+++ b/tests/simple_interlock_client.c
@@ -194,11 +194,11 @@ CHEAT_DECLARE(
cheat_assert(MSTRO_OK==mstro_cdo_attribute_set(
e->cdo,
MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
- &s->size));
+ &s->size, true));
cheat_assert(MSTRO_OK==mstro_cdo_attribute_set(
e->cdo,
MSTRO_ATTR_CORE_CDO_RAW_PTR,
- e->buf));
+ e->buf, false));
fprintf(stderr, "declared %s, handle %x\n",
e->name, e->cdo);
break;
diff --git a/tests/simple_telemetry_listener.c b/tests/simple_telemetry_listener.c
index 3a3b7e70513fb17e5bbb466383a48a9bb4c800ab..7ae3372cd2b012329e24bae70525daa26a95369d 100644
--- a/tests/simple_telemetry_listener.c
+++ b/tests/simple_telemetry_listener.c
@@ -212,7 +212,7 @@ parse_arguments(int argc, char **argv)
}
if(g_verbose) {
- fprintf(stderr, "Configuration: %s/%s/%s/%llu/%d\n",
+ fprintf(stderr, "Configuration: %s/%s/%s/%" PRIu64 "/%d\n",
g_conf_workflow_name, g_conf_component_name,
g_conf_terminate_after, g_conf_max_wait, g_conf_logdst);
}
diff --git a/tests/test_attributes.yaml b/tests/test_attributes.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..68008c9aa1da68dd0df39002d20a627537a127f4
--- /dev/null
+++ b/tests/test_attributes.yaml
@@ -0,0 +1,16 @@
+# A user-defined schema. The minimum is name and version
+schema-name: Test Attributes
+schema-version: 0
+
+schema-namespace: ".maestro.test."
+
+# attributes section is optional; if given it needs to have a sequence value
+maestro-attributes:
+
+# Top-level attributes
+
+ - key: "t_1"
+ type: str()
+ required: False
+ default: ""
+ documentation: Value length is the attribute size
diff --git a/transport/gfs.c b/transport/gfs.c
index 7d3090c46c3f17152c5609ec7b6cc64b1b17a36a..c3b12490ba3e9d800654b5902b664bc273ab8f70 100644
--- a/transport/gfs.c
+++ b/transport/gfs.c
@@ -69,7 +69,7 @@ mstro_transport_gfs_src_execute(mstro_cdo src, Mstro__Pool__TransferTicket* tick
FILE *f = NULL;
char* path = ticket->gfs->path;
- INFO("Path src app will use for transport: %s\n", path);
+ DEBUG("Path src app will use for transport: %s\n", path);
if (path == NULL) {
ERR("Ticket for GFS transport does not contain a path\n");
return MSTRO_INVARG;
@@ -77,21 +77,29 @@ mstro_transport_gfs_src_execute(mstro_cdo src, Mstro__Pool__TransferTicket* tick
f = fopen(path,"w");
if (f == NULL) {
- ERR("Failed to open %s for GFS transport\n", path);
+ ERR("Failed to open %s for GFS transport (errno: %d -- %s)\n",
+ path, errno, strerror(errno));
return MSTRO_FAIL;
}
- if (fwrite(dl.data, sizeof(char), dl.len/sizeof(char), f) != dl.len) {
- ERR("Partial write on %s for GFS transport\n", path);
- return MSTRO_FAIL;
+RETRY_GFS_TRANSPORT_WRITE: ;
+ size_t bytes_written = fwrite(dl.data, sizeof(char), dl.len, f);
+ if (bytes_written != dl.len) {
+ if (errno == EAGAIN)
+ goto RETRY_GFS_TRANSPORT_WRITE;
+ ERR("Partial write on %s (buf: %p) for GFS transport (errno: %d -- %s)\n",
+ path, dl.data, errno, strerror(errno));
+ return MSTRO_FAIL;
}
if (fclose(f) != 0) {
- ERR("Failed to close %s for GFS transport\n", path);
+ ERR("Failed to close %s for GFS transport (errno: %d -- %s)\n",
+ path, errno, strerror(errno));
return MSTRO_FAIL;
}
- INFO("src app successfully executed transport CDO %s\n", src->name);
+ DEBUG("src app successfully executed transport CDO %s\n",
+ src->name);
return MSTRO_OK;
}
@@ -109,10 +117,10 @@ mstro_transport_gfs_dst_execute(mstro_cdo dst,
INFO("Executing gfs transport dst side for CDO %s\n", dst->name);
mstro_status status;
- int64_t len = ticket->data_size;
+ size_t len = (size_t)ticket->data_size;
char* data;
- DEBUG("Incoming CDO, size %" PRIi64 "\n", len);
+ DEBUG("Incoming CDO, size %zu\n", len);
status = mstro_transport_get_dst_buffer(dst, (void*)&data);
if(status!=MSTRO_OK) {
@@ -125,25 +133,30 @@ mstro_transport_gfs_dst_execute(mstro_cdo dst,
/* data is the raw-ptr or freshly allocated, CDO attribute is correct */
FILE *f = NULL;
char* path = ticket->gfs->path;
- INFO("Path src app will use for transport: %s\n", path);
if (path == NULL) {
ERR("Ticket for GFS transport does not contain a path\n");
return MSTRO_INVARG;
}
+ DEBUG("Path dst app will use for transport: %s\n", path);
f = fopen(path,"rb");
if (f == NULL) {
ERR("Failed to open %s for GFS transport: %d (%s)\n",
path, errno, strerror(errno));
return MSTRO_FAIL;
}
-
- if (fread(data, sizeof(char), len, f) != (size_t)len) {
- ERR("Partial read on %s for GFS transport\n", path);
+RETRY_GFS_TRANSPORT_READ: ;
+ size_t bytes_read = fread(data, sizeof(char), len, f);
+ if (bytes_read != (size_t)len) {
+ if (errno == EAGAIN)
+ goto RETRY_GFS_TRANSPORT_READ;
+ ERR("Partial read (%d out of %d bytes) on %s for GFS transport (errno: %d -- %s)\n",
+ bytes_read, len, path, errno, strerror(errno));
return MSTRO_FAIL;
}
if (fclose(f) != 0) {
- ERR("Failed to close %s for GFS transport\n", path);
+ ERR("Failed to close %s for GFS transport (errno: %d -- %s)\n",
+ path, errno, strerror(errno));
return MSTRO_FAIL;
}
@@ -170,7 +183,7 @@ mstro_transport_gfs_dst_execute(mstro_cdo dst,
/* abort(); */
/* } */
- INFO("dst app successfully executed transport CDO %s\n", dst->name);
+ DEBUG("dst app successfully executed transport CDO %s\n", dst->name);
if (! (ticket->gfs->keep_file)) {
int s = unlink(ticket->gfs->path);
@@ -178,7 +191,7 @@ mstro_transport_gfs_dst_execute(mstro_cdo dst,
WARN("Couldn't unlink file used for gfs transfer (%s): %d (%s)\n",
ticket->gfs->path, s, strerror(s));
} else {
- INFO("Removed the transfer file from GFS\n");
+ DEBUG("Removed the transfer file\n");
}
}
diff --git a/transport/transport.c b/transport/transport.c
index 230260565a21e2c389e83f271cb4e4a8e69cd571..0690af7edff5ead202d06172de6cb67b7c0c930f 100644
--- a/transport/transport.c
+++ b/transport/transport.c
@@ -43,6 +43,8 @@
#include "maestro/i_cdo.h"
#include <inttypes.h>
+#include <sys/stat.h>
+
#include "transport/transport_gfs.h"
#ifdef HAVE_MIO
@@ -170,11 +172,63 @@ mstro_transport__src_datalen_get(mstro_cdo src,
return MSTRO_OK;
}
+/** find next separator in PATH. Return the start of that segment */
+static inline
+const char * next_sep (const char *path)
+{
+ while (*path)
+ if (*path == '/' || *path == '\\')
+ return path;
+ else
+ path++;
+ return NULL;
+}
+
+/** ensure all directories in DIRNAME exist. If not, try creating them */
+static inline
+mstro_status
+mkdirhier(char *dirname)
+{
+ char buf[PATH_MAX];
+ const char *prev = dirname;
+ const char *next;
+ struct stat sb;
+
+ while(NULL!=(next = next_sep(prev))) {
+ strncpy(buf, dirname, next - dirname) ;
+ buf[next - dirname] = '\0';
+ if(stat(buf,&sb)) {
+ mkdir(buf, 0777); /* umask taken into account by system */
+ /* we ignore errors, as we'll see an error on the last part */
+ }
+ prev=next+1;
+ }
+ /* handle last part */
+ if(stat(dirname, &sb)) {
+ int s = mkdir(dirname, 0777);
+ if(s!=0) {
+ ERR("Failed to create GFS transport directory %s: %d (%s)\n",
+ dirname, errno, strerror(errno));
+ return MSTRO_FAIL;
+ }
+ }
+ return MSTRO_OK;
+}
+
mstro_status
mstro_transport_init()
{
srand(time(NULL)); /* We'll need that to generate random IDs */
+
+ /* ensure GFS dir is set up */
+ mstro_status s = mkdirhier(g_mstro_transport_gfs_dir);
+ if(s!=MSTRO_OK) {
+ ERR("Failed to initialize GFS transport. Check your setting of %s\n",
+ MSTRO_ENV_TRANSPORT_GFS_DIR);
+ return s;
+ }
+
#ifdef HAVE_MIO
/* MIO */
if(g_mio_available) {
@@ -317,9 +371,32 @@ mstro_transport_execute(
if (cdo->state & MSTRO_CDO_STATE_OFFERED) {
/* we are called on the source side of transport */
{
- /* FIXME: should check this for robustness: */
- DEBUG("Not checking that SRC cdo has expected size (%" PRIi64 ")\n",
- ticket->data_size);
+ /* let's check ticket size matches local CDO size */
+ enum mstro_cdo_attr_value_type type;
+ const void *sizep=NULL;
+ int64_t src_size;
+ mstro_status s = mstro_attribute_dict_get(cdo->attributes,
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE,
+ &type, &sizep, NULL, false);
+ if(s==MSTRO_OK) {
+ src_size=*(int64_t*)sizep;
+ } else if(s==MSTRO_NOENT && type==MSTRO_CDO_ATTR_VALUE_INVALID) {
+ ERR("Attribute not found or value invalid\n");
+ src_size=-1;
+ } else {
+ ERR("Failed to look up %s (%d: %s)\n",
+ MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE, s, mstro_status_description(s));
+ return MSTRO_FAIL;
+ }
+
+ if (ticket->data_size == 0) {
+ DEBUG("0-transport\n");
+ }
+ else if (src_size != ticket->data_size) {
+ ERR("SRC cdo has a size (%" PRIi64 ") that doesn't match ticket-specified size (%" PRIi64 ")\n",
+ src_size, ticket->data_size);
+ return MSTRO_FAIL;
+ }
}
mstro_status (*f)(mstro_cdo src, Mstro__Pool__TransferTicket *t)
= g_transport_registry[ticket->method].src_func;