fix: argo params issue with char encodings (#2635)

introduce a script for parsing param values from a base64 encoding
instead of passing raw values to the command line, in order to get
around some encoding issues

other JSONType parameter fixes
- support both json-strings and proper objects as parameter default
values
- support json-stringified values in argo event payloads
- unify param serialisation between cron and manual triggers

Author: saikonen

metaflow/plugins/argo/argo_workflows.py

@@ -609,7 +609,16 @@ def _process_parameters(self):
             # the JSON equivalent of None to please argo-workflows. Unfortunately it
             # has the side effect of casting the parameter value to string null during
             # execution - which needs to be fixed imminently.
-            if not is_required or default_value is not None:
+            if default_value is None:
+                default_value = json.dumps(None)
+            elif param_type == "JSON":
+                if not isinstance(default_value, str):
+                    # once to serialize the default value if needed.
+                    default_value = json.dumps(default_value)
+                # adds outer quotes to param
+                default_value = json.dumps(default_value)
+            else:
+                # Make argo sensors happy
                 default_value = json.dumps(default_value)
 
             parameters[param.name] = dict(
@@ -941,11 +950,7 @@ def _compile_workflow_template(self):
                     Arguments().parameters(
                         [
                             Parameter(parameter["name"])
-                            .value(
-                                "'%s'" % parameter["value"]
-                                if parameter["type"] == "JSON"
-                                else parameter["value"]
-                            )
+                            .value(parameter["value"])
                             .description(parameter.get("description"))
                             # TODO: Better handle IncludeFile in Argo Workflows UI.
                             for parameter in self.parameters.values()
@@ -2054,7 +2059,7 @@ def _container_templates(self):
                         # {{foo.bar['param_name']}}.
                         # https://argoproj.github.io/argo-events/tutorials/02-parameterization/
                         # http://masterminds.github.io/sprig/strings.html
-                        "--%s={{workflow.parameters.%s}}"
+                        "--%s=\\\"$(python -m metaflow.plugins.argo.param_val {{=toBase64(workflow.parameters['%s'])}})\\\""
                         % (parameter["name"], parameter["name"])
                         for parameter in self.parameters.values()
                     ]
@@ -3842,37 +3847,27 @@ def _compile_sensor(self):
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
                                                 # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" %s) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}%s{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
                                                 % (
-                                                    v,
                                                     v,
                                                     (
-                                                        "| toRawJson | squote"
+                                                        '{{- $pv:=(get $.Input.body.payload "%s") -}}{{ if kindIs "string" $pv }}{{- $pv | toRawJson -}}{{- else -}}{{ $pv | toRawJson | toRawJson }}{{- end -}}'
+                                                        % v
                                                         if self.parameters[
                                                             parameter_name
                                                         ]["type"]
                                                         == "JSON"
-                                                        else "| toRawJson"
+                                                        else '{{- (get $.Input.body.payload "%s" | toRawJson) -}}'
+                                                        % v
                                                     ),
                                                 ),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem
                                                 # to be any way for us to skip
-                                                value=(
-                                                    json.dumps(
-                                                        self.parameters[parameter_name][
-                                                            "value"
-                                                        ]
-                                                    )
-                                                    if self.parameters[parameter_name][
-                                                        "type"
-                                                    ]
-                                                    == "JSON"
-                                                    else self.parameters[
-                                                        parameter_name
-                                                    ]["value"]
-                                                ),
+                                                value=self.parameters[parameter_name][
+                                                    "value"
+                                                ],
                                             )
                                             .dest(
                                                 # this undocumented (mis?)feature in

metaflow/plugins/argo/param_val.py

@@ -0,0 +1,19 @@
+import sys
+import base64
+import json
+
+
+def parse_parameter_value(base64_value):
+    val = base64.b64decode(base64_value).decode("utf-8")
+
+    try:
+        return json.loads(val)
+    except json.decoder.JSONDecodeError:
+        # fallback to using the original value.
+        return val
+
+
+if __name__ == "__main__":
+    base64_val = sys.argv[1]
+
+    print(parse_parameter_value(base64_val))