diff --git a/attributes/maestro-schema.c b/attributes/maestro-schema.c
index c03d39143b9aad55bf85339cf91dac1287caaf29..5232a0b3a588224ff777640425a6efa9b684b9c2 100644
--- a/attributes/maestro-schema.c
+++ b/attributes/maestro-schema.c
@@ -996,12 +996,14 @@ mstro_schema_lookup_attribute(mstro_schema schema,
/** attribute entry in dictionary (hashable by key) */
struct mstro_attribute_entry_ {
- UT_hash_handle hh; /** hashable by key */
- mstro_symbol key; /** the key (interned attribute name) */
- void *val; /** a value, to be interpreted by looking up the
- * expected type of the attribute in the
- * appropriate schema */
- size_t valsize; /** allocated space for val */
+ UT_hash_handle hh; /**< hashable by key */
+ mstro_symbol key; /**< the key (interned attribute name) */
+ void *val; /**< a value, to be interpreted by looking up the
+ * expected type of the attribute in the
+ * appropriate schema */
+ size_t valsize; /**< allocated space for val */
+ bool user_owned_val; /**< whether the val allocation is owned by the
+ * user (if not we must free it eventually) */
/* FIXME: this is the place where serialized versions of the entry
* should be cached if needed */
/* these may be unset -- this can be checked by comparing the string
@@ -1247,7 +1249,76 @@ BAILOUT:
return s;
}
+
+static inline
+mstro_status
+mstro_attribute_entry_create(mstro_symbol sym, struct mstro_attribute_entry_ **result)
+{
+ if(sym==NULL)
+ return MSTRO_INVARG;
+ if(result==NULL)
+ return MSTRO_INVOUT;
+ *result = malloc(sizeof(struct mstro_attribute_entry_));
+ if(*result==NULL) {
+ ERR("Failed to allocate attribute entry\n");
+ return MSTRO_NOMEM;
+ }
+ (*result)->key = sym;
+ (*result)->val = NULL;
+ (*result)->valsize = 0;
+ (*result)->user_owned_val = false;
+ (*result)->serialized_yaml = NULL;
+ mstro__pool__aval__init(& ((*result)->serialized_pb));
+ return MSTRO_OK;
+}
+
+static inline
+mstro_status
+mstro_attribute_entry_dispose(struct mstro_attribute_entry_ *entry)
+{
+ if(entry==NULL)
+ return MSTRO_INVARG;
+ mstro_status status=MSTRO_OK;
+ if(! entry->user_owned_val) {
+ free(entry->val);
+ }
+ if(entry->serialized_yaml)
+ free(entry->serialized_yaml);
+
+ switch(entry->serialized_pb.val_case) {
+ case MSTRO__POOL__AVAL__VAL_STRING:
+ free(entry->serialized_pb.string);
+ break;
+ case MSTRO__POOL__AVAL__VAL_BYTES:
+ if(entry->serialized_pb.bytes.len)
+ free(entry->serialized_pb.bytes.data);
+ break;
+ case MSTRO__POOL__AVAL__VAL_TIMESTAMP:
+ mstro__pool__timestamp__free_unpacked(
+ entry->serialized_pb.timestamp, NULL);
+ break;
+ case MSTRO__POOL__AVAL__VAL__NOT_SET:
+ case MSTRO__POOL__AVAL__VAL_BOOL:
+ case MSTRO__POOL__AVAL__VAL_INT32:
+ case MSTRO__POOL__AVAL__VAL_INT64:
+ case MSTRO__POOL__AVAL__VAL_UINT32:
+ case MSTRO__POOL__AVAL__VAL_UINT64:
+ case MSTRO__POOL__AVAL__VAL_FLOAT:
+ case MSTRO__POOL__AVAL__VAL_DOUBLE:
+ /* These are inlined, no need to free them */
+ break;
+ default:
+ ERR("Unhandled protobuf AVAL type: %d\n",
+ entry->serialized_pb.val_case);
+ status=MSTRO_FAIL;
+ }
+ free(entry);
+
+ return status;
+}
+
+
static inline
mstro_status
mstro_attributes__parse_helper(yaml_parser_t parser,
@@ -1255,6 +1326,7 @@ mstro_attributes__parse_helper(yaml_parser_t parser,
mstro_attribute_dict *result)
{
mstro_status status = MSTRO_UNIMPL;
+ bool result_allocated_here=false;
struct partial_key *keystack =NULL;
if(schema==NULL) {
ERR("Can't parse without schema\n");
@@ -1264,13 +1336,16 @@ mstro_attributes__parse_helper(yaml_parser_t parser,
ERR("NULL attribute dict\n");
return MSTRO_INVOUT;
}
- *result = malloc(sizeof(struct mstro_attribute_dict_));
if(*result==NULL) {
- ERR("Failed to allocate attribute dict\n");
- return MSTRO_NOMEM;
+ *result = malloc(sizeof(struct mstro_attribute_dict_));
+ if(*result==NULL) {
+ ERR("Failed to allocate attribute dict\n");
+ return MSTRO_NOMEM;
+ }
+ (*result)->dict=NULL;
+ (*result)->schema=schema; /* for now, will be added when successful */
+ result_allocated_here = true;
}
- (*result)->dict=NULL;
- (*result)->schema=NULL; /* for now, will be added when successful */
yaml_event_t event;
@@ -1396,28 +1471,38 @@ mstro_attributes__parse_helper(yaml_parser_t parser,
}
}
/* it's a valid attribute */
- struct mstro_attribute_entry_ *entry = malloc(sizeof(struct mstro_attribute_entry_));
- if(entry==NULL) {
- ERR("Failed to allocate attribute entry\n");
- status=MSTRO_NOMEM;
- goto BAILOUT;
+ mstro_attribute_entry entry=NULL;
+ bool new_entry=false;
+
+ DEBUG("looking up sym %p in dict\n", decl->key_symbol);
+ HASH_FIND(hh, (*result)->dict, &(decl->key_symbol), sizeof(mstro_symbol), entry);
+ if(entry) {
+ DEBUG("Replacing value of previously set attribute |%s|\n", keystack->fqkey);
+ if(! entry->user_owned_val) {
+ free(entry->val);
+ entry->val=NULL;
+ }
+ } else {
+ new_entry=true;
+ status = mstro_attribute_entry_create(decl->key_symbol, &entry);
+ if(status!=MSTRO_OK)
+ goto BAILOUT;
}
- entry->key = decl->key_symbol;
- entry->serialized_yaml = NULL;
- entry->serialized_pb.val_case = MSTRO__POOL__AVAL__VAL__NOT_SET;
status = mstro_attributes_parse_val(schema, decl, val, val_len, entry);
if(status!=MSTRO_OK) {
ERR("Failed to parse |%s| as value for attribute |%s|\n",
val, mstro_symbol_name(decl->key_symbol));
+ if(new_entry)
+ mstro_attribute_entry_dispose(entry);
goto BAILOUT;
} else {
DEBUG("Parsed |%s| as valid value for attribute |%s|\n",
val, mstro_symbol_name(decl->key_symbol));
/* FIXME: describe_entry function call here */
}
-
- HASH_ADD(hh, (*result)->dict, key, sizeof(mstro_symbol), entry);
+ if(new_entry)
+ HASH_ADD(hh, (*result)->dict, key, sizeof(mstro_symbol), entry);
DEBUG("Handled entry for %s, cleaning keystack\n", keystack->fqkey);
free(keystack->fqkey);
@@ -1550,7 +1635,11 @@ mstro_attributes__parse_helper(yaml_parser_t parser,
/* depth); */
/* } */
-BAILOUT:
+BAILOUT:
+ if(status!=MSTRO_OK && result_allocated_here) {
+ free(*result);
+ }
+
return status;
}
@@ -1570,8 +1659,7 @@ mstro_attributes_parse(mstro_schema schema,
goto BAILOUT;
}
if(yaml_fragment==NULL) {
- *result=NULL;
- s=MSTRO_OK;
+ s=MSTRO_INVARG;
goto BAILOUT;
}
@@ -1592,7 +1680,7 @@ mstro_attributes_parse(mstro_schema schema,
if(s!=MSTRO_OK) {
ERR("Failed to parse attribute yaml string |%s|\n", yaml_fragment);
} else {
- INFO("Parsed yaml attribute string, %zu entries\n", HASH_COUNT((*result)->dict));
+ INFO("Parsed yaml attribute string, now %zu entries in dictionary\n", HASH_COUNT((*result)->dict));
(*result)->schema = schema;
}
@@ -1600,13 +1688,58 @@ BAILOUT:
return s;
}
+mstro_status
+mstro_attribute_dict_get_schema(mstro_attribute_dict dict, mstro_schema *schema_p)
+{
+ if(dict==NULL)
+ return MSTRO_INVARG;
+ if(schema_p==NULL)
+ return MSTRO_INVOUT;
+ *schema_p=dict->schema;
+ return MSTRO_OK;
+}
+
+mstro_status
+mstro_attribute_dict_set_defaults(mstro_schema schema,
+ bool override,
+ mstro_attribute_dict *result)
+{
+ if(schema==NULL)
+ return MSTRO_INVARG;
+ if(result==NULL)
+ return MSTRO_INVOUT;
+ if(*result==NULL) {
+ *result = malloc(sizeof(struct mstro_attribute_dict_));
+ if(*result==NULL)
+ return MSTRO_NOMEM;
+ (*result)->schema = schema;
+ (*result)->dict = NULL;
+ }
+
+ WARN("Not filling in default values in dictionary\n");
+ return MSTRO_OK;
+}
+
+
mstro_status
-mstro_attributes_default(mstro_schema schema,
- bool override,
- mstro_attribute_dict *result)
+mstro_attribute_dict_dispose(mstro_attribute_dict dict)
{
- return MSTRO_UNIMPL;
+ if(dict==NULL)
+ return MSTRO_INVARG;
+ /* schema will be refcounted one day ...*/
+ mstro_status status = MSTRO_OK;
+
+ struct mstro_attribute_entry_ *el,*tmp;
+ HASH_ITER(hh,dict->dict,el,tmp) {
+ DEBUG("Deleting attribute %s from dict %p\n",
+ mstro_symbol_name(el->key), dict->dict);
+ HASH_DELETE(hh,dict->dict,el);
+ status = status | mstro_attribute_entry_dispose(el);
+ }
+
+ free(dict);
+ return status;
}
mstro_status
@@ -1820,15 +1953,9 @@ mstro_attribute_dict_set(mstro_attribute_dict dict, const char *key,
}
/* create fresh entry */
- entry = malloc(sizeof(struct mstro_attribute_entry_));
- if(entry==NULL) {
- ERR("Failed to allocate attribute entry\n");
- status=MSTRO_NOMEM;
+ status = mstro_attribute_entry_create(decl->key_symbol, &entry);
+ if(status!=MSTRO_OK)
goto BAILOUT;
- }
- entry->key = decl->key_symbol;
- entry->serialized_yaml = NULL;
- entry->serialized_pb.val_case = MSTRO__POOL__AVAL__VAL__NOT_SET;
/* FIXME: setting the val size should be part of the default init for entry */
/* FIXME: entry should have the tdecl->parsed_type-> kind as a slot in it. */
@@ -1946,6 +2073,7 @@ mstro_attribute_dict_set(mstro_attribute_dict dict, const char *key,
}
WARN("Not checking type restrictions\n");
entry->val = val;
+ entry->user_owned_val = true;
status=MSTRO_OK;
diff --git a/attributes/maestro-schema.h b/attributes/maestro-schema.h
index 94044c27eb503660865e8e4890f3870ab0a29b42..fc74f5dad86bf39489e5dd24433c75b238c0274e 100644
--- a/attributes/maestro-schema.h
+++ b/attributes/maestro-schema.h
@@ -123,9 +123,11 @@ typedef struct mstro_attribute_dict_ * mstro_attribute_dict;
** specified for a CDO. The input will be parsed as though it were
** normalized to a depth-1 sequence of 'key: value' entries. Keys
** will be looked up as attribute keys in the @arg schema, and values
- ** will be parsed accordingly. The result is a sequence of entries
- ** suitable for merging into the attribute dictionary of a CDO, or
- ** serializing for pool messages.
+ ** will be parsed accordingly.
+ **
+ ** If *result == NULL a fresh dictionay will be allocated.
+ ** If *result != NULL the parsed values will be stored in the dictionary, replacing
+ ** any previously set values for the same keys.
**
** Example:
** maestro.core.cdo:
@@ -137,6 +139,7 @@ typedef struct mstro_attribute_dict_ * mstro_attribute_dict;
** maestro.core.cdo.persist.cdo.persist: "on"
** maestro.core.cdo.scope.size: 1024
**
+ ** @arg yaml_fragment may not be NULL.
**/
mstro_status
@@ -146,13 +149,26 @@ mstro_attributes_parse(mstro_schema schema,
/** Inject default values from schema into attribute dictionary.
+ **
+ ** If *result==NULL, allocate a fresh, default-filled, attribute
+ ** dictionary. Otherwise just (re-)set the default values from the
+ ** schema.
**
** When @arg override is true, overrides existing entries, otherwise only fills in missing ones.
**/
mstro_status
-mstro_attributes_default(mstro_schema schema,
- bool override,
- mstro_attribute_dict *result);
+mstro_attribute_dict_set_defaults(mstro_schema schema,
+ bool override,
+ mstro_attribute_dict *result);
+
+/** Deallocate a dictionary.
+ *
+ */
+mstro_status
+mstro_attribute_dict_dispose(mstro_attribute_dict dict);
+
+mstro_status
+mstro_attribute_dict_get_schema(mstro_attribute_dict dict, mstro_schema *schema_p);
/** Look up maestro attribute in dictionary.
diff --git a/include/maestro/attributes.h b/include/maestro/attributes.h
index ba282d8f33398c15747b851fcdb7a2a34f0df05a..ed2d3e64c904ac45579b6dfdf5c19d660d8c937f 100644
--- a/include/maestro/attributes.h
+++ b/include/maestro/attributes.h
@@ -56,11 +56,11 @@ extern "C" {
** This is the Attribute API, as developed for D3.2
**/
-/** Default attribute value specifier for a CDO */
-#define MSTRO_ATTR_DEFAULT NULL
+/* /\** a structure holding a set of attributes *\/ */
+typedef struct mstro_attribute_dict_* mstro_cdo_attributes;
-/** a structure holding a set of attributes */
-typedef struct mstro_cdo_decl_attr_ *mstro_cdo_decl_attr;
+/** Default attribute value specifier for a CDO */
+#define MSTRO_ATTR_DEFAULT ((mstro_cdo_attributes)NULL)
/**@defgroup MSTRO_Attr_builtin Maestro Core Predefined Attributes
@@ -242,6 +242,31 @@ enum mstro_cdo_attr_value_type {
MSTRO_CDO_ATTR_VALUE__MAX
};
+
+ // All data types have an implicit length, so we need a wrapper
+/** A data type to hold blobs.
+ **
+ ** This provides a wrapper around the user memory region so that we can know the size of it.
+ **/
+typedef struct {
+ size_t len; /**< the length of the data at @ref mstro_blob.data */
+ void *data; /**< an opaque memory region of size @ref mstro_blob.len */
+} mstro_blob;
+
+/** convenience function to wrap a blob and its size into a mstro_blob object.
+ *
+ * (You can also use stack-allocated mstro_blob structures, or allocate them yourself.)
+ *
+ * If you create a blob with this function you need to free it using mstro_blob_dispose()
+ */
+mstro_status
+mstro_blob_create(size_t len, void *data, mstro_blob **result_p);
+
+/** convenience function to dispose a blob allocated with mstro_blob_create()
+ */
+mstro_status
+mstro_blob_dispose(mstro_blob *b);
+
/** Opaque CDO handle. */
typedef struct mstro_cdo_ *mstro_cdo;
diff --git a/include/maestro/cdo.h b/include/maestro/cdo.h
index 82eb3cde416c9fe1862673d88ca9914b24495f28..5ae0f24e90aca2f05c24009b82394a4ce82eeacb 100644
--- a/include/maestro/cdo.h
+++ b/include/maestro/cdo.h
@@ -90,7 +90,7 @@ mstro_cdo_name(mstro_cdo cdo);
**/
mstro_status
mstro_cdo_declare(const char *name,
- mstro_cdo_decl_attr attributes,
+ mstro_cdo_attributes attributes,
mstro_cdo *result);
diff --git a/include/maestro/i_cdo.h b/include/maestro/i_cdo.h
index 9977f0b0274d807217fba6ccdc2c498ee149ee4e..b125087c5aeb0afe8aa953dffbf9d9f7b9ea5450 100644
--- a/include/maestro/i_cdo.h
+++ b/include/maestro/i_cdo.h
@@ -287,9 +287,8 @@ struct mstro_cdo_ {
struct mstro_cdo_id gid; /**< the globally unique ID */
/* all other attributes and name and id are stored in the following table */
- struct mstro_cdo_attr_table_* attributes;
- char* attributes_yaml; /**< attributes in yaml to be parsed once at seal time (cyaml limitations) */
- char* attributes_namespace; /**< namespace string; default: ".maestro.core.cdo" */
+ mstro_cdo_attributes attributes; /**< attribute and schema information */
+ char* current_namespace; /**< namespace string; default: ".maestro.core.cdo" */
};
/* cdo.h has typedef struct mstro_cdo_ * mstro_cdo; */
diff --git a/include/maestro/i_globals.h b/include/maestro/i_globals.h
index 9455fa962c80aef74fd84093c7d1148b08dcbc26..c475db59b07782dec8e00de14ae3fc75e8353551 100644
--- a/include/maestro/i_globals.h
+++ b/include/maestro/i_globals.h
@@ -39,6 +39,7 @@
#include "maestro.h"
#include "maestro/i_state.h"
#include "protocols/mstro_pool.pb-c.h"
+#include "attributes/maestro-schema.h"
#include <stdbool.h>
#include <stdatomic.h>
#include <pthread.h>
@@ -103,4 +104,7 @@ extern Mstro__Pool__Apptoken g_pool_apptoken;
/** the app ID (packed version of @ref g_pool_app_id) used in communicating with the pool manager. */
extern Mstro__Pool__Appid g_pool_appid;
+/** the fundamental built-in attribute schema. Filled early in mstro_core_init(), then constant */
+extern mstro_schema g_mstro_core_schema_instance;
+
#endif /* MAESTRO_I_GLOBALS_H_ */
diff --git a/maestro/attributes.c b/maestro/attributes.c
index caf509ea379a75f677dd60785e302aea17cf21e3..575062d0c9f55c6790593f08fc32aef5ea34ed43 100644
--- a/maestro/attributes.c
+++ b/maestro/attributes.c
@@ -29,6 +29,7 @@
#include "maestro/attributes.h"
#include "maestro/i_attributes.h"
#include "maestro/logging.h"
+#include "attributes/maestro-schema.h"
#include "c-timestamp/timestamp.h"
@@ -98,75 +99,33 @@ mstro_cdo_attribute_set(mstro_cdo cdo, const char* key, void* val)
ERR("Invalid key: not a cstring\n");
return MSTRO_INVARG;
}
- /* FIXME Absolute path supported only for default namespace yet */
- if ( strncmp (key,MSTRO_CDO_ATTR_NAMESPACE_DEFAULT, strlen(MSTRO_CDO_ATTR_NAMESPACE_DEFAULT))
- && !strncmp (key,MSTRO_CDO_ATTR_NAMESPACE_DEFAULT, 1)) {
- ERR("set attribute on CDO \"%s\" failed because absolute path supported only for default namespace yet\n", cdo->name);
- return MSTRO_INVARG;
- }
+
if(mstro_cdo_state_check(cdo, MSTRO_CDO_STATE_SEALED)) {
ERR("cannot set attributes on sealed CDO\n");
return MSTRO_FAIL;
}
- /* Search key by pointer FIXME: do for all other attributes*/
- if (key == MSTRO_ATTR_CORE_CDO_RAW_PTR) {
- cdo->raw_ptr = val;
- goto SUCCESS;
- }
- if (key == MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY) {
- cdo->mamba_array = val;
- goto SUCCESS;
- }
-
- if (key == MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE) {
- cdo->attributes->scope.local_size = *(uint64_t*)val;
-
- /* we rely on being able to overwrite all data at SEAL time with the YAML-parse result, so need to store this setting */
- char scope_string[128];
- snprintf(scope_string, 128,
- "scope:\n"
- " local_size: %" PRIu64 , *(uint64_t*)val);
- if (! (MSTRO_OK == mstro_cdo_attribute_set_yaml(cdo, scope_string))) {
- ERR("Couldn't set attribute yaml for CDO %s\n", cdo->name);
- return MSTRO_FAIL;
- }
-
- DEBUG("Set local-size (interned key) to %"PRIu64"\n", cdo->attributes->scope.local_size);
- goto SUCCESS;
- }
-
- /* If not found, search by string FIXME: do for all other attributes*/
- if (!strcmp(key, MSTRO_ATTR_CORE_CDO_RAW_PTR)) {
- cdo->raw_ptr = val;
- goto SUCCESS;
- }
- if (!strcmp(key, MSTRO_ATTR_CORE_CDO_RAW_PTR)) {
- cdo->mamba_array = val;
- goto SUCCESS;
- }
- if (!strcmp(key, MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE)) {
- cdo->attributes->scope.local_size = *(uint64_t*)val;
- char scope_string[128];
- snprintf(scope_string, 128,
- "scope:\n"
- " local_size: %" PRIu64, *(uint64_t*)val);
- if (! (MSTRO_OK == mstro_cdo_attribute_set_yaml(cdo, scope_string))) {
- ERR("Couldn't set attribute yaml for CDO %s\n", cdo->name);
- return MSTRO_FAIL;
- }
- cdo->attributes->scope.local_size = *(uint64_t*)val;
- DEBUG("Set local-size (string key) to %"PRIu64"\n",
- cdo->attributes->scope.local_size);
- goto SUCCESS;
+ char *tmpfqkey=NULL;
+ const char *fqkey=NULL;
+ if(key[0]!='.') {
+ /* qualify it */
+ tmpfqkey = malloc(strlen(key)+strlen(cdo->current_namespace));
+ if(tmpfqkey==NULL) {
+ ERR("Cannot allocate for fully-qualified key\n");
+ return MSTRO_NOMEM;
+ }
+ fqkey = tmpfqkey;
+ } else {
+ fqkey=key;
}
- /* No match */
- WARN("No match for attribute \"%s\"\n", key);
- return MSTRO_INVARG;
-
-SUCCESS:
- return MSTRO_OK;
+ mstro_status status
+ = mstro_attribute_dict_set(cdo->attributes, fqkey,
+ MSTRO_CDO_ATTR_VALUE_INVALID, /* we dont help in checking */
+ val);
+ if(tmpfqkey)
+ free(tmpfqkey);
+ return status;
}
mstro_status
@@ -202,18 +161,16 @@ mstro_cdo_attribute_set_default(mstro_cdo cdo)
{
if (cdo == NULL)
return MSTRO_INVARG;
- if (cdo->attributes == NULL)
- return MSTRO_INVARG;
+ /* we can rely on the dict being non-NULL (at least the core schema is loaded) */
- char tmp_attributes_yaml [] =
- #include "cdo-attributes-default.txt"
- ;
- cdo->attributes_yaml = malloc(sizeof(char) * sizeof(tmp_attributes_yaml));
- if(cdo->attributes_yaml == NULL)
- return MSTRO_NOMEM;
- strcpy(cdo->attributes_yaml, tmp_attributes_yaml);
-
- return MSTRO_OK;
+ mstro_schema schema;
+ mstro_status s= mstro_attribute_dict_get_schema(cdo->attributes, &schema);
+ if(s!=MSTRO_OK)
+ return s;
+
+ return mstro_attribute_dict_set_defaults(schema,
+ true,
+ &(cdo->attributes));
}
mstro_status
@@ -221,24 +178,20 @@ mstro_cdo_attribute_set_yaml(mstro_cdo cdo, const char* keyval_in_yaml)
{
if(cdo==NULL || keyval_in_yaml==NULL)
return MSTRO_INVARG;
- if (cdo->attributes_yaml == NULL)
- return MSTRO_INVARG;
- if(mstro_cdo_state_check(cdo, MSTRO_CDO_STATE_SEALED))
+ if(mstro_cdo_state_check(cdo, MSTRO_CDO_STATE_SEALED)) {
+ ERR("Cannot add yaml attributes on SEALED CDO\n");
return MSTRO_FAIL;
+ }
- char *tmp = realloc(cdo->attributes_yaml,
- strlen(cdo->attributes_yaml) + strlen(keyval_in_yaml)
- + BYTE_ASCII_LF + BYTE_ASCII_NUL); // Not pretty, 1 additional byte for '\n' we cat right after and 1 byte for '\0' not accounted for by strlen
- if(tmp==NULL)
- return MSTRO_NOMEM;
- else
- cdo->attributes_yaml = tmp;
-
- strcat(cdo->attributes_yaml, keyval_in_yaml);
- strcat(cdo->attributes_yaml, "\n");
-
- return MSTRO_OK;
+ mstro_schema schema;
+ mstro_status s= mstro_attribute_dict_get_schema(cdo->attributes, &schema);
+ if(s!=MSTRO_OK)
+ return s;
+
+ return mstro_attributes_parse(schema,
+ keyval_in_yaml,
+ &cdo->attributes);
}
mstro_status
@@ -325,3 +278,26 @@ mstro_timestamp_to_tm_local(const mstro_timestamp *tsp, struct tm *tmp)
+mstro_status
+mstro_blob_create(size_t len, void *data, mstro_blob **result_p)
+{
+ if(result_p==NULL)
+ return MSTRO_INVOUT;
+ *result_p = malloc(sizeof(mstro_blob));
+ if(!*result_p)
+ return MSTRO_NOMEM;
+ (*result_p)->len = len;
+ (*result_p)->data = data;
+ return MSTRO_OK;
+}
+
+/** convenience function to dispose a blob allocated with mstro_blob_create()
+ */
+mstro_status
+mstro_blob_dispose(mstro_blob *b)
+{
+ if(b==NULL)
+ return MSTRO_INVARG;
+ free(b);
+ return MSTRO_OK;
+}
diff --git a/maestro/cdo.c b/maestro/cdo.c
index 3ee65835953a8d1cc94cd61eacb173c2a756efd4..9ddebf5ae363fb355db69bc3b6d19f405a7f5d66 100644
--- a/maestro/cdo.c
+++ b/maestro/cdo.c
@@ -133,7 +133,13 @@ mstro_cdo__alloc(void)
/* initialize parts that need special handling */
atomic_init(&(res->state), MSTRO_CDO_STATE_INVALID);
/* add attributes space */
- assert(MSTRO_OK==mstro_cdo_attr_table__alloc(&(res->attributes)));
+ mstro_status s = mstro_attribute_dict_set_defaults(g_mstro_core_schema_instance,
+ false,
+ &(res->attributes));
+ if(s!=MSTRO_OK) {
+ ERR("Cannot create CDO, aborting\n");
+ abort();
+ }
return res;
}
@@ -149,16 +155,14 @@ mstro_cdo__free(mstro_cdo *cdoptr)
if((*cdoptr)->name)
free((*cdoptr)->name);
- status = mstro_cdo_attr_table__destroy((*cdoptr)->attributes);
+ status = mstro_attribute_dict_dispose((*cdoptr)->attributes);
if(status!=MSTRO_OK) {
ERR("Failed to destroy CDO attribute table\n");
goto BAILOUT;
}
- if((*cdoptr)->attributes_yaml)
- free((*cdoptr)->attributes_yaml);
- if((*cdoptr)->attributes_namespace)
- free((*cdoptr)->attributes_namespace);
+ if((*cdoptr)->current_namespace)
+ free((*cdoptr)->current_namespace);
if((*cdoptr)->mamba_array) {
mmbError stat = mmb_array_destroy((*cdoptr)->mamba_array);
@@ -240,7 +244,7 @@ mstro_cdo_name(mstro_cdo cdo)
mstro_status
mstro_cdo_declare(const char *name,
- mstro_cdo_decl_attr attributes,
+ mstro_cdo_attributes attributes,
mstro_cdo *result)
{
if(name==NULL) {
@@ -252,6 +256,7 @@ mstro_cdo_declare(const char *name,
return MSTRO_INVOUT;
}
if(attributes!=MSTRO_ATTR_DEFAULT) {
+ ERR("non-default declaration attributes unsupported, FIXME\n");
return MSTRO_UNIMPL;
}
@@ -341,14 +346,12 @@ mstro_cdo_declare(const char *name,
return s;
/* TODO initialize default namespace*/
- (*result)->attributes_namespace = malloc(sizeof(char)*128);
- if ((*result)->attributes_namespace == NULL) {
- ERR("Cannot allocate for CDO namespace\n");
+ (*result)->current_namespace = strdup(MSTRO_CDO_ATTR_NAMESPACE_DEFAULT);
+ if ((*result)->current_namespace == NULL) {
+ ERR("Cannot allocate for default CDO namespace\n");
mstro_cdo__free(result);
return MSTRO_NOMEM;
}
- sprintf((*result)->attributes_namespace, "%s", MSTRO_CDO_ATTR_NAMESPACE_DEFAULT);
-
WITH_CDO_ID_STR(idstr,&(*result)->id,
INFO("Declared CDO `%s', (local ID: %s)\n",
@@ -379,12 +382,14 @@ mstro_cdo_declaration_seal(mstro_cdo cdo)
}
- /* Parse once and for all attributes yaml string. */
- status = mstro_cdo_attributes_parse_string(cdo);
- if (status != MSTRO_OK) {
- ERR("CDO `%s` yaml attribute parse failed\n", cdo->name);
- goto BAILOUT;
- }
+ /* /\* Parse once and for all attributes yaml string. *\/ */
+ /* status = mstro_cdo_attributes_parse_string(cdo); */
+ /* if (status != MSTRO_OK) { */
+ /* ERR("CDO `%s` yaml attribute parse failed\n", cdo->name); */
+ /* goto BAILOUT; */
+ /* } */
+ void *raw_ptr;
+ mmbArray *mamba_array;
if(cdo->raw_ptr!=NULL && cdo->mamba_array!=NULL) {
ERR("CDO `%s` has both raw_ptr and existing mamba_array, unsupported\n",
@@ -473,7 +478,7 @@ mstro_cdo_declaration_seal(mstro_cdo cdo)
/* FIXME: this should use the fixed data but a serialized version
* of the current attributes */
WARN("Sealing with silly attributes\n");
- attr.yaml_string = cdo->attributes_yaml;
+ //xx// attr.yaml_string = cdo->attributes_yaml;
Mstro__Pool__Seal seal = MSTRO__POOL__SEAL__INIT;
seal.cdoid = &cdoid;
diff --git a/maestro/cdo_attributes_schema.c b/maestro/cdo_attributes_schema.c
index bb83d96b86758c3084023fead12a79926c48c370..207b7b988abdb100ca46076b81bf78235e2be6e1 100644
--- a/maestro/cdo_attributes_schema.c
+++ b/maestro/cdo_attributes_schema.c
@@ -93,47 +93,7 @@ static const cyaml_config_t config = {
.mem_fn = cyaml_mem, /* Use the default memory allocator. */
};
-mstro_status
-mstro_cdo_attributes_parse_string(mstro_cdo cdo)
-{
- if (cdo == NULL)
- return MSTRO_INVARG;
-
- struct mstro_cdo_attr_table_* good_table;
- struct mstro_cdo_attr_table_ _tmp_table;
- struct mstro_cdo_attr_table_* tmp_table = &_tmp_table;
- good_table = cdo->attributes;
-
- cyaml_err_t err;
-// err = cyaml_load_file("/cray/css/users/chaine/projects/maestro/maestro-core/maestro/cdo-attributes-default.yaml", &config, &top_schema, (cyaml_data_t **)&tmp_table, NULL); // works now
- err = cyaml_load_data((const uint8_t*)cdo->attributes_yaml, strlen(cdo->attributes_yaml), &config, &top_schema, (cyaml_data_t**)&tmp_table, NULL); //&seq_count_out);
-
- if (err != CYAML_OK) {
- fprintf(stderr, "cyaml: %s\n", cyaml_strerror(err));
- return MSTRO_FAIL;
- }
- // XXX Here we need a copy, because cyaml would throw away our attribute allocation/defaults, can't memcpy either because structures don't match
- // FIXME can do it slightly less hardcoded with a loop and data_offset's, or maybe partial memcpy's...
- good_table->allocate_now = tmp_table->allocate_now;
- good_table->maestro_provided_storage = tmp_table->maestro_provided_storage;
- good_table->name = (unsigned char*)strdup((const char*)tmp_table->name);
- good_table->level = tmp_table->level;
- good_table->id = (unsigned char*)strdup((const char*)tmp_table->id);
- good_table->lifetime = (unsigned char*)strdup((const char*)tmp_table->lifetime);
- good_table->redundancy = (unsigned char*)strdup((const char*)tmp_table->redundancy);
- good_table->persist = tmp_table->persist;
- good_table->desist = tmp_table->desist;
-
- good_table->scope.local_size = tmp_table->scope.local_size;
-// good_table->scope.layout.regular_1d.element_size = tmp_table->scope.layout.regular_1d.element_size;
- good_table->scope.distribution = (unsigned char*)strdup((const char*)tmp_table->scope.distribution);
-
- // cyaml cleanup
- cyaml_free(&config, &top_schema, tmp_table, 0);
-
- return MSTRO_OK;
-}
enum mstro_cdo_attr_value_type
mstro_cdo_attr__match_type_cyaml(enum cyaml_type t, uint32_t size)
@@ -194,100 +154,100 @@ mstro_cdo_attr__match_key_schema(const cyaml_schema_field_t s[], size_t len, con
return -1;
}
-mstro_status
-mstro_cdo_attr_table__lookup(mstro_cdo cdo,
- const char *key,
- enum mstro_cdo_attr_value_type *valtype,
- void **value_dst)
-{
-// XXX how to navigate through the attribute table levels using the schema table?
-// -> I will hardcode this for now
- const cyaml_schema_field_t* s;
- void* ptr_tab, *ptr_val;
- int found_top, found_scope, i;
- size_t valsize;
-
- // Match key param with <WILDCARD>schema[].key
- found_top = mstro_cdo_attr__match_key_schema(top_mapping_schema, sizeof(top_mapping_schema)/sizeof(cyaml_schema_field_t)-1, key);
- if (found_top > -1) {
- s = top_mapping_schema;
- ptr_tab = (void*)(cdo->attributes);
- i = found_top;
- }
- else {
- found_scope = mstro_cdo_attr__match_key_schema(scope_mapping_schema, sizeof(scope_mapping_schema)/sizeof(cyaml_schema_field_t)-1, key);
- if (found_scope > -1) {
- s = scope_mapping_schema;
- ptr_tab = (void*)&(cdo->attributes->scope);
- i = found_scope;
- }
- }
- if (found_top == -1 && found_scope == -1) {
- /* Search key by pointer FIXME: do for all other attributes*/
- if (key == MSTRO_ATTR_CORE_CDO_RAW_PTR) {
- ERR("%s cannot be queried, use mstro_cdo_access_ptr()\n",
- MSTRO_ATTR_CORE_CDO_RAW_PTR);
- return MSTRO_INVARG;
- }
- if (key == MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY) {
- ERR("% cannot be queried, use mstro_cdo_access_mamba_array()\n",
- MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY);
- return MSTRO_INVARG;
- }
- if (key == MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE) {
- *value_dst = (void*)&cdo->attributes->scope.local_size;
- goto SUCCESS;
- }
+/* mstro_status */
+/* mstro_cdo_attr_table__lookup(mstro_cdo cdo, */
+/* const char *key, */
+/* enum mstro_cdo_attr_value_type *valtype, */
+/* void **value_dst) */
+/* { */
+/* // XXX how to navigate through the attribute table levels using the schema table? */
+/* // -> I will hardcode this for now */
+/* const cyaml_schema_field_t* s; */
+/* void* ptr_tab, *ptr_val; */
+/* int found_top, found_scope, i; */
+/* size_t valsize; */
+
+/* // Match key param with <WILDCARD>schema[].key */
+/* found_top = mstro_cdo_attr__match_key_schema(top_mapping_schema, sizeof(top_mapping_schema)/sizeof(cyaml_schema_field_t)-1, key); */
+/* if (found_top > -1) { */
+/* s = top_mapping_schema; */
+/* ptr_tab = (void*)(cdo->attributes); */
+/* i = found_top; */
+/* } */
+/* else { */
+/* found_scope = mstro_cdo_attr__match_key_schema(scope_mapping_schema, sizeof(scope_mapping_schema)/sizeof(cyaml_schema_field_t)-1, key); */
+/* if (found_scope > -1) { */
+/* s = scope_mapping_schema; */
+/* ptr_tab = (void*)&(cdo->attributes->scope); */
+/* i = found_scope; */
+/* } */
+/* } */
+/* if (found_top == -1 && found_scope == -1) { */
+/* /\* Search key by pointer FIXME: do for all other attributes*\/ */
+/* if (key == MSTRO_ATTR_CORE_CDO_RAW_PTR) { */
+/* ERR("%s cannot be queried, use mstro_cdo_access_ptr()\n", */
+/* MSTRO_ATTR_CORE_CDO_RAW_PTR); */
+/* return MSTRO_INVARG; */
+/* } */
+/* if (key == MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY) { */
+/* ERR("% cannot be queried, use mstro_cdo_access_mamba_array()\n", */
+/* MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY); */
+/* return MSTRO_INVARG; */
+/* } */
+/* if (key == MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE) { */
+/* *value_dst = (void*)&cdo->attributes->scope.local_size; */
+/* goto SUCCESS; */
+/* } */
- /* If not found, search by string FIXME: do for all other attributes*/
- if (!strcmp(key, MSTRO_ATTR_CORE_CDO_RAW_PTR)) {
- ERR("%s cannot be queried, use mstro_cdo_access_ptr()\n",
- MSTRO_ATTR_CORE_CDO_RAW_PTR);
- return MSTRO_INVARG;
- }
- if (!strcmp(key, MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY)) {
- ERR("%s cannot be queried, use mstro_cdo_access_mamba_array()\n",
- MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY);
- return MSTRO_INVARG;
- }
- if (!strcmp(key, MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE)) {
- *value_dst = (void*)&cdo->attributes->scope.local_size;
- goto SUCCESS;
- }
-
- /* not found */
- *valtype = MSTRO_CDO_ATTR_VALUE_NA;
- *value_dst = NULL;
- return MSTRO_INVARG;
- }
-
- // Set valtype param with the right maestro cdo attribute type corresponding to the cyaml_type for <WILDCARD>schema[].key
- *valtype = mstro_cdo_attr__match_type_cyaml(s[i].value.type, s[i].value.data_size);
-
- // Match <WILDCARD>schema[].key with its value in attribute table, using data_offset
- ptr_val = (void*)((char*)ptr_tab + s[i].data_offset);
-
- if (MSTRO_CDO_ATTR_VALUE_cstring == *valtype){
- ptr_val = *(void**)ptr_val;
-// valsize = strlen((char*)ptr_val);
- }
-/* else
- valsize = s[i].value.data_size;
-
- if (MSTRO_CDO_ATTR_VALUE_cstring == *valtype)
- *value_dst = malloc(valsize+1);
- else
- *value_dst = malloc(valsize);
- if (*value_dst == NULL)
- return MSTRO_NOMEM;
-
- memcpy(*value_dst, ptr_val, valsize);
- if (MSTRO_CDO_ATTR_VALUE_cstring == *valtype)
- ((char*)(*value_dst))[valsize] = '\0';
-*/
- *value_dst = ptr_val;
-
-SUCCESS:
- return MSTRO_OK;
-}
+/* /\* If not found, search by string FIXME: do for all other attributes*\/ */
+/* if (!strcmp(key, MSTRO_ATTR_CORE_CDO_RAW_PTR)) { */
+/* ERR("%s cannot be queried, use mstro_cdo_access_ptr()\n", */
+/* MSTRO_ATTR_CORE_CDO_RAW_PTR); */
+/* return MSTRO_INVARG; */
+/* } */
+/* if (!strcmp(key, MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY)) { */
+/* ERR("%s cannot be queried, use mstro_cdo_access_mamba_array()\n", */
+/* MSTRO_ATTR_CORE_CDO_MAMBA_ARRAY); */
+/* return MSTRO_INVARG; */
+/* } */
+/* if (!strcmp(key, MSTRO_ATTR_CORE_CDO_SCOPE_LOCAL_SIZE)) { */
+/* *value_dst = (void*)&cdo->attributes->scope.local_size; */
+/* goto SUCCESS; */
+/* } */
+
+/* /\* not found *\/ */
+/* *valtype = MSTRO_CDO_ATTR_VALUE_NA; */
+/* *value_dst = NULL; */
+/* return MSTRO_INVARG; */
+/* } */
+
+/* // Set valtype param with the right maestro cdo attribute type corresponding to the cyaml_type for <WILDCARD>schema[].key */
+/* *valtype = mstro_cdo_attr__match_type_cyaml(s[i].value.type, s[i].value.data_size); */
+
+/* // Match <WILDCARD>schema[].key with its value in attribute table, using data_offset */
+/* ptr_val = (void*)((char*)ptr_tab + s[i].data_offset); */
+
+/* if (MSTRO_CDO_ATTR_VALUE_cstring == *valtype){ */
+/* ptr_val = *(void**)ptr_val; */
+/* // valsize = strlen((char*)ptr_val); */
+/* } */
+/* /\* else */
+/* valsize = s[i].value.data_size; */
+
+/* if (MSTRO_CDO_ATTR_VALUE_cstring == *valtype) */
+/* *value_dst = malloc(valsize+1); */
+/* else */
+/* *value_dst = malloc(valsize); */
+/* if (*value_dst == NULL) */
+/* return MSTRO_NOMEM; */
+
+/* memcpy(*value_dst, ptr_val, valsize); */
+/* if (MSTRO_CDO_ATTR_VALUE_cstring == *valtype) */
+/* ((char*)(*value_dst))[valsize] = '\0'; */
+/* *\/ */
+/* *value_dst = ptr_val; */
+
+/* SUCCESS: */
+/* return MSTRO_OK; */
+/* } */
diff --git a/maestro/core.c b/maestro/core.c
index 5e3480abfdb881f843902d48c79cbc84e41aba1e..d7ad811e3a1ae7de059595cf69a063481cfca2b6 100644
--- a/maestro/core.c
+++ b/maestro/core.c
@@ -71,6 +71,14 @@ mstro_core_init(const char *workflow_name,
return MSTRO_NOMEM;
}
}
+ status=mstro_schema_parse(MSTRO_SCHEMA_BUILTIN_YAML_CORE,
+ MSTRO_SCHEMA_BUILTIN_YAML_CORE_LEN,
+ &g_mstro_core_schema_instance);
+ if(status!=MSTRO_OK) {
+ ERR("Failed to parse built-in core schema\n");
+ goto BAILOUT;
+ }
+
DEBUG("mstro_core_init: %s/%s/% "PRIi64 " in thread %" PRIxPTR" complete\n",
data->workflow_name, data->component_name, data->component_index,
(intptr_t)pthread_self());
diff --git a/maestro/globals.c b/maestro/globals.c
index cb44ea38502439b9f4e795744ac8a2f256b214a4..02a95be71e6d50afe45f2ceb0496e9563f9b0c86 100644
--- a/maestro/globals.c
+++ b/maestro/globals.c
@@ -70,3 +70,6 @@ Mstro__Pool__Apptoken g_pool_apptoken = MSTRO__POOL__APPTOKEN__INIT;
/** re-usable app id structure */
Mstro__Pool__Appid g_pool_appid = MSTRO__POOL__APPID__INIT;
+
+/** the fundamental built-in schema. Filled early in mstro_core_init(), then constant */
+mstro_schema g_mstro_core_schema_instance = NULL;
diff --git a/protocols/mstro_pool.pb-c.h b/protocols/mstro_pool.pb-c.h
index 8816b13f0b21bcb025272f67f797f9512ae0b5f4..2e7f3b0afbb7ced8c95aa68d26ccf9c95a4221b5 100644
--- a/protocols/mstro_pool.pb-c.h
+++ b/protocols/mstro_pool.pb-c.h
@@ -405,7 +405,7 @@ struct _Mstro__Pool__AVal
double double_;
char *string;
/*
- * protobuf limits this to 2^32
+ * protobuf limits this to 2^32; this is used for mstro_blob attributes
*/
ProtobufCBinaryData bytes;
Mstro__Pool__Timestamp *timestamp;
diff --git a/protocols/mstro_pool.proto b/protocols/mstro_pool.proto
index 5320db4d7c5428fb1dab64ea98b35b68fbae50a4..0c3e2c8eadd66f94b3b3684e70a3afee277ca0c3 100644
--- a/protocols/mstro_pool.proto
+++ b/protocols/mstro_pool.proto
@@ -155,7 +155,7 @@ message AVal {
double double = 7;
string string = 8;
- bytes bytes = 9; /* protobuf limits this to 2^32 */
+ bytes bytes = 9; /* protobuf limits this to 2^32; this is used for mstro_blob attributes */
Timestamp timestamp = 10;
};
};
diff --git a/tests/check_schema_parse.c b/tests/check_schema_parse.c
index 5fbe1a19ecf2c6c2c11a01405001c9a09e5845da..90cb51c368b66fb48176d61b815c439919fe5139 100644
--- a/tests/check_schema_parse.c
+++ b/tests/check_schema_parse.c
@@ -67,8 +67,8 @@
CHEAT_TEST(core_schema_parse,
- mstro_schema s1, s2, s3;
- mstro_attribute_dict dict;
+ mstro_schema s1=NULL, s2=NULL, s3=NULL;
+ mstro_attribute_dict dict=NULL;
mstro_status s=MSTRO_OK;
s=mstro_schema_parse_from_file(USER_YAML, &s2);
@@ -109,9 +109,9 @@ CHEAT_TEST(core_schema_parse,
)
CHEAT_TEST(core_schema_parse_builtin,
- mstro_schema s1, s2;
+ mstro_schema s1=NULL, s2=NULL;
- mstro_attribute_dict dict;
+ mstro_attribute_dict dict=NULL;
mstro_status s=MSTRO_OK;
cheat_assert(MSTRO_OK==mstro_schema_parse(MSTRO_SCHEMA_BUILTIN_YAML_CORE,
@@ -176,10 +176,10 @@ CHEAT_TEST(core_schema_parse_builtin,
mstro_attribute_dict_set(dict, ecmwfdate, MSTRO_CDO_ATTR_VALUE_INVALID, &ts));
/* and query the TS */
- void *tsval;
+ const void *tsval;
cheat_assert(MSTRO_OK==
mstro_attribute_dict_get(dict, ecmwfdate, &type, &tsval, NULL));
- mstro_timestamp *tsval_cast = (mstro_timestamp*)tsval;
+ const mstro_timestamp *tsval_cast = (const mstro_timestamp*)tsval;
cheat_assert(tsval_cast->sec==ts.sec && tsval_cast->nsec==ts.nsec && tsval_cast->offset==ts.offset);