Work on YAML schema parsing and core schema

.gitignore

@@ -68,3 +68,4 @@ maestro/cdo-attributes-default.yaml
 tests/cdo1
 tests/cdo2
 attributes/attributes.html
+tests/check_schema_parse

Makefile.am

@@ -58,7 +58,8 @@ libmaestro_la_SOURCES =
 libmaestro_la_LIBADD = 			 \
 	maestro/libmaestro_core.la    \
 	protocols/libmaestro_proto.la \
-	transport/libmaestro_transport.la
+	transport/libmaestro_transport.la \
+	attributes/libattributes.la
 
 # README
 dist_doc_DATA = README.md

attributes/Makefile.am

@@ -40,7 +40,7 @@ libattributes_la_CPPFLAGS = \
 		-I$(top_srcdir)/deps/libyaml/include \
 		-I$(top_srcdir)/deps/libcyaml/include
 
-libattributes_la_SOURCES = 
+libattributes_la_SOURCES = attribute_schema.c
 
 libattributes_la_LIBADD=$(top_builddir)/deps/libyaml/src/libyaml.la 
 

attributes/attribute_schema.c

+/* -*- mode:c -*- */
+/** @file
+ ** @brief Maestro Attribute Schema ingest and data structures
+ **/
+/*
+ * Copyright (C) 2020 Cray Computer GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "maestro/status.h"
+#include "maestro/logging.h"
+#include "maestro/i_uthash.h"
+#include "protocols/mstro_pool.pb-c.h"
+
+#include "yaml.h"
+
+#include <errno.h>
+#include <stdlib.h>
+
+/* A single attribute definition */
+struct mstro_attribute_entry {
+  UT_hash_handle *hh; /**< hashable, by key, using magic hash function
+                       * below */
+  char *key; /**< attribute key, also key in hash table */
+  /** The type permitted for this attribute */
+  Mstro__Pool__AVal__ValCase val_type;
+};
+  
+/* Data structure for (a fragment of) a schema */
+struct mstro_attribute_schema {
+  /** The name of this schema */
+  char *name;
+  /** Its version number */
+  size_t version;
+  /** the set of declarations in this schema */
+  struct mstro_attribite_entry *declarations;
+};
+
+/* The (merge of) all loaded schemata. This is conceptually read-only
+ * after mstro_init(), and holds the information on all defined
+ * attribute keys and value types */
+static
+struct mstro_attribute_schema *g_mstro_attribute_schema = NULL;
+
+
+/** Initialize schema module */
+mstro_status
+mstro_schema_init(void)
+{
+  return MSTRO_OK;
+}
+
+/** finalize schema module */
+mstro_status
+mstro_schema_finalize(void)
+{
+  if(g_mstro_attribute_schema!=NULL) {
+    WARN("Not deallocating schema tree\n");
+  }
+  return MSTRO_OK;
+}
+
+static inline
+mstro_status
+mstro_schema__parse(yaml_parser_t parser,
+                    struct mstro_attribute_schema **result)
+{
+  yaml_token_t  token;
+  char *last_key=NULL;
+  enum {
+    INVALID,
+    KEY, VALUE,
+    BLOCKSEQ, BLOCKENT, BLOCKMAP
+  } last_kind = INVALID;
+  ssize_t depth = -1;
+
+  *result = malloc(sizeof(struct mstro_attribute_schema));
+  if(*result == NULL) {
+    return MSTRO_NOMEM;
+  }
+
+  /* for now, let's see if token-based scanning is sufficient */
+  do {
+    yaml_parser_scan(&parser, &token);
+    switch(token.type) {
+      /* Stream start/end */
+      case YAML_STREAM_START_TOKEN:
+        DEBUG("STREAM START\n");
+        depth++;
+        break;
+      case YAML_STREAM_END_TOKEN:
+        DEBUG("STREAM END\n");
+        depth--;
+        break;
+
+        /* Token types (read before actual token) */
+      case YAML_KEY_TOKEN:
+        DEBUG("(Key token)   \n");
+        last_kind = KEY;
+        break;
+      case YAML_VALUE_TOKEN:
+        DEBUG("(Value token) \n");
+        last_kind = VALUE;
+        break;
+
+        /* Block delimeters */
+      case YAML_BLOCK_SEQUENCE_START_TOKEN:
+        DEBUG("Start Block (Sequence)\n");
+        last_kind = BLOCKSEQ;
+        depth++;
+        break;
+      case YAML_BLOCK_ENTRY_TOKEN:
+        DEBUG("Start Block (Entry)\n");
+        last_kind = BLOCKENT;
+        break;
+      case YAML_BLOCK_END_TOKEN:
+        DEBUG("End block\n");
+        depth--;
+        break;
+        
+        /* Data */
+      case YAML_BLOCK_MAPPING_START_TOKEN:
+        DEBUG("[Block mapping]\n");
+        last_kind=BLOCKMAP;
+        depth++;
+        break;
+      case YAML_SCALAR_TOKEN:
+        DEBUG("scalar %s (for kind %d)\n",
+              token.data.scalar.value, last_kind);
+        /* yaml_char_t is signed */
+        const char *val = (const char*)token.data.scalar.value;
+        switch(last_kind) {
+          case KEY:
+            /* store key for value coming up */
+            if(last_key!=NULL) {
+              ERR("two keys in succession: stack %s, here %s\n",
+                  last_key, val);
+              free(last_key);
+            }
+            last_key = strdup(val);
+            if(last_key==NULL) {
+              ERR("Failed to allocate key\n");
+            }
+            break;
+          case VALUE:
+            if(last_key==NULL) {
+              ERR("No key preceeded this value\n");
+            } else {
+              /* This is where the built-in meta-schema is hiding */
+              if(strcmp(last_key, "schema-name")==0) {
+                (*result)->name = last_key;
+                last_key=NULL;
+              } else if(strcmp(last_key, "schema-version")==0) {
+                (*result)->version = atol(val);
+                free(last_key);
+              } else {
+                DEBUG("Value |%s| for unknown key |%s|\n", val, last_key);
+                free(last_key);
+              }
+              last_key=NULL;
+            }
+            break;
+          default:
+            WARN("Unexpected last_kind value\n");
+        }          
+        break;
+        
+        /* Others */
+      default:
+        WARN("Got token of type %d\n", token.type);
+    }
+    if(token.type != YAML_STREAM_END_TOKEN)
+      yaml_token_delete(&token);
+  } while(token.type != YAML_STREAM_END_TOKEN);
+  yaml_token_delete(&token);
+
+  if(depth!=-1) {
+    ERR("YAML stream nesting does not balance, depth %u at end (should be -1)\n",
+        depth);
+  }
+
+  
+  return MSTRO_UNIMPL;
+}
+
+/* merge SCHEMA into global schema tree. Consume argument, even on error */
+static inline
+mstro_status
+mstro_schema__merge(struct mstro_attribute_schema *schema)
+{
+  return MSTRO_UNIMPL;
+}
+
+
+/** Add a file's content to schema tree */
+mstro_status
+mstro_schema_add_from_file(char *path)
+{
+  mstro_status s;
+  if(path==NULL) {
+    s=MSTRO_INVARG;
+    goto BAILOUT;
+  }
+  
+  FILE *f = fopen(path, "r");
+  if(f==NULL) {
+    ERR("Failed to open schema file %s: %d (%s)\n",
+        path, errno, strerror(errno));
+    s=MSTRO_FAIL;
+    goto BAILOUT;
+  }
+  
+  /* Initialize parser */
+  yaml_parser_t parser;
+  if(!yaml_parser_initialize(&parser)) {
+    ERR("Failed to initialize YAML parser\n");
+    s= MSTRO_FAIL;
+    goto BAILOUT_CLOSE;
+  }
+
+  /* Set input file */
+  yaml_parser_set_input_file(&parser, f);
+
+  struct mstro_attribute_schema *schema;
+  s=mstro_schema__parse(parser, &schema);
+  yaml_parser_delete(&parser);
+  
+  if(s!=MSTRO_OK) {
+    ERR("Failed to parse schema %s\n", path);
+  } else {
+    INFO("Parsed schema definition %s\n", schema->name);
+    /* merge it (consumes argument, even on error) */
+    s=mstro_schema__merge(schema);
+    if(s!=MSTRO_OK) {
+      ERR("Failed to merge schema\n");
+    }
+  }
+     
+BAILOUT_CLOSE:
+  fclose(f);
+BAILOUT:
+  return s;
+}  

attributes/attribute_schema.h

+/* -*- mode:c -*- */
+/** @file
+ ** @brief Maestro CDO Attributes Schema handling
+ **/
+/*
+ * Copyright (C) 2020 Cray Computer GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MAESTRO_I_ATTRIBUTE_SCHEMA_H_
+#define MAESTRO_I_ATTRIBUTE_SCHEMA_H_ 1
+
+
+/** Add a file's content to schema tree */
+mstro_status
+mstro_schema_add_from_file(char *path);
+
+
+#endif 
+

attributes/maestro-core.yaml

-# This is the Maestro Core YAML schema
+# This is the Maestro Core YAML schema, maestro-core.yaml
+# Make sure you verify it using maestro-schema-schema.yaml
 schema-name: Maestro Core
 schema-version: 0
 
-maestro-schema:
-  - key: ".maestro.core.cdo.name"
-    type: str(regex)    
-    required: true
+# relative attributes will be grafted below this root:
+schema-namespace: ".maestro.core"
 
+# Values permitted in user-defined types used
+schema-type-values:
+  # values occuring in lifetime-strictness flag
+  - &lifetime-strictness-values enum('optional', 'required')
 
+# user-defined types used in this schema
+schema-types:
+  
+  -  &lifetime-interval
+     name: CDO lifetime type
+     typespec: map()
+
+  -  &lifetime-strictness
+     name: CDO lifetime specifier interpretation
+     typespec: *lifetime-strictness-values
+
+
+# Individual attribute descriptions
+maestro-attributes:
+  ## TOPLEVEL attributes
+
+  - key: ".maestro.core.cdo.name" # note: this is an absolute attribute name
+    type: str()
+    required: True
+    documentation: The unique name of the CDO
+
+  # note: the ID is not user-visible; it's synthetized during a workflow
+
+  - key: "cdo.allocate-now" # note: this is a relative attribute name
+    type: bool()
+    required: False
+    default: False
+    documentation: Perform immediate allocation for the CDO
+
+  - key: "cdo.persist"
+    type: bool()
+    required: False
+    default: False
+    documentation: Ensure a persistent copy of the CDO is preserved
+                   when the last copy of the CDO is WITHDRAWN from the
+                   pool. (Note the non-trivial semantic interaction
+                   with the cdo.desist attribute.)
+
+  - key: "cdo.desist"
+    type: bool()
+    required: False
+    default: True
+    documentation: Ensure that all persistent copies of the CDO are
+                   removed from persistent storage when the last
+                   reference to the CDO is WITHDRAWn from the
+                   pool. (Note the non-trivial semantic interaction
+                   with the cdo.desist attribute.)
+
+  ## Lifetime related
+  - key: "cdo.lifetime"
+    type: *lifetime-interval
+    required: False
+    documentation: CDO lifetime specification.
+  - key: "cdo.lifetime-strictness"
+    type: *lifetime-strictness
+    required: False
+    default: *lifetime-strictness-values
+    documentation: CDO lifetime specification.
+
+    
+  ## SCOPE related
+  - key: "cdo.scope.size"
+    type: int(min=-1)
+    required: False
+    default: -1
+    documentation: The total size of the CDO. -1 indicates Unknown/Unallocated.
+
+  # Layouts
+  # Distributions
+
+
+# (end of maestro-core.yaml)
 

attributes/maestro-schema-schema.yaml

-# This is the schema that every maestro attribute schema must be
-# validated against.
+## This is the schema that every maestro attribute schema must be
+## validated against.
 
+# Every schema needs to have a descriptive name
 schema-name: str()
+# ... and a version number
 schema-version: int(min=0)
 
+# It may define a namespace prefix to permit using shorter (relative)
+# attribute names in the definitions later
+schema-namespace: regex('(^\.$)|(^(\.[^\s\.]+)+\.?$)', 
+                        required=False,
+                        none=False, multiline=False,
+                        name='fully-qualified attribute namestring')
+
+# If non-elementary types are to be used in the attribute definition these must be declared
+schema-types: list(include('maestro-user-type-def'),
+                   required=False)
+
+schema-type-values: list(include('maestro-user-typespec'),
+                    required=False)
+          
 # A Maestro schema needs to be a valid yaml document
-maestro-schema: List(include('maestro-schema-node'))
+maestro-attributes: list(include('maestro-attribute-def'), required=False)
 ---
-maestro-schema-node:
-  key: str()
+maestro-attribute-def:
+  key: regex('(^(\.[^\s\.]+)+$)|(^[^\s\.]+(\.[^\s\.]+)*$)',
+             required=False,
+             none=False, multiline=False,
+             name='valid attribute namestring')
+  type: include('maestro-user-typespec')
   required: bool()
-  type: str()
+  default: Any(include('maestro-builtin-typeval'),
+               include('maestro-user-typeval'),
+               required=False, none=False)
+  documentation: str(multiline=True)
+
+
+
+---
+maestro-user-type-def:
+  name: str()
+  typespec: include('maestro-user-typespec')
 
 
+---
+maestro-builtin-typespec: any(regex('^str\(.*\)$'),
+                              regex('^bool\(.*\)$'),
+                              regex('^int\(.*\)$'),
+                              regex('^map\(.*\)$'),
+                              regex('^enum\(.*\)$'),
+                              regex('^map\(.*\)$')
+                              )
+
+---
+maestro-builtin-typeval:  Any(bool(),
+                              int(),
+                              enum('None'))
+  
+---
+# FIXME: this needs better inside typing
+maestro-user-typespec: any(include('maestro-builtin-typespec'),
+                           map(),
+                           list())
+---
+# things that can be checked directly
+# plus
+# things that can be run-time checked
+maestro-user-typeval: any(include('maestro-builtin-typeval'), 
+                          include('maestro-builtin-typespec') 
+                          )
                           

tests/Makefile.am

@@ -33,7 +33,7 @@
 #LOG_COMPILER=valgrind --leak-check=full
 AM_DEFAULT_SOURCE_EXT = .c
 AM_LDFLAGS = -no-install
-AM_CFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/deps/mamba -I$(top_srcdir)/deps/libcyaml/include -I$(top_srcdir)/protocols -I$(top_srcdir)/transport
+AM_CFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/deps/mamba -I$(top_srcdir)/deps/libcyaml/include -I$(top_srcdir)/protocols -I$(top_srcdir)/transport -I$(top_srcdir)/attributes -DTOPSRCDIR=$(top_srcdir)
 LDADD = $(top_builddir)/libmaestro.la
 
 check_HEADERS = cheat.h
@@ -43,6 +43,7 @@ TESTS = check_version check_init\
 	check_protobuf_c \
 	check_transport_gfs \
         check_declare \
+	check_schema_parse \
 	check_pool_local \
 	check_pool_local_putget \
 	check_pool_local_stress \
@@ -63,6 +64,7 @@ check_PROGRAMS = check_version check_init \
 		 check_protobuf_c \
 		check_transport_gfs \
                  check_declare \
+		 check_schema_parse \
                  check_pool_local \
                  check_pool_local_stress \
 		 check_pool_local_putget \

tests/check_schema_parse.c

+/* check schema parsing  */
+/*
+ * Copyright (C) 2020 Cray Computer GmbH
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* needed before inclusion of cheat.h: */
+#ifndef __BASE_FILE__
+#define __BASE_FILE__ __FILE__
+#endif
+
+#include "cheat.h"
+
+#include "maestro.h"
+#include <string.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <errno.h>
+
+#include "attribute_schema.h"
+
+#ifndef TOPSRCDIR
+#error TOPSRCDIR needs to be defined to compile this so that we can find the yaml schema sources
+#endif
+
+#define XSTRINGIFY(s) #s
+#define STRINGIFY(s) XSTRINGIFY(s) 
+
+#define CORE_YAML STRINGIFY(TOPSRCDIR) "/attributes/maestro-core.yaml"
+#define USER_YAML STRINGIFY(TOPSRCDIR) "/attributes/user.yaml"
+
+
+
+CHEAT_TEST(core_schema_parse,
+           cheat_assert(MSTRO_OK==mstro_schema_add_from_file(CORE_YAML));
+           cheat_assert(MSTRO_OK==mstro_schema_add_from_file(USER_YAML));
+           )
+