Commit 6c071163 authored by Sigbjørn Revheim's avatar Sigbjørn Revheim

Fixed handling of option lists

parent 7de09a24
Pipeline #22964 failed with stages
in 1 minute and 19 seconds
......@@ -309,16 +309,19 @@ filelist
;
options
: <","> option+
: <","> (option | numlist_option)+
;
option
: ID (<"("> option_param ( <","> option_param ) * <")">)?
: ID (<"("> option_param ( <","> option_param)* <")">)?
;
numlist_option
: ID <"("> numlist <")">
;
option_param
: varlist
| numlist
| string
| display_format
;
......@@ -341,7 +344,7 @@ varRange
;
numlist
: (number | range)+
: (number | range)+ ( <","> (number | range)+ )*
;
local_command
......
(ns ^{
:author "Ole Voldsæter"
:author "Ole Voldsæter, Sigbjørn Revheim"
:doc "Translator STATA -> SDTL"
} stata2sdtl.core
(:require
......@@ -36,18 +36,19 @@
;; mapping of grammar production names to transformation functions
(def transformations
{
:options (fn [& args] (reduce #(assoc %1 (keyword (first %2)) (apply concat (second %2))) {} args))
:option (fn [name & args] [name args])
:option_param (fn [& args] args)
:options (fn [& args] (into {} args))
:option (fn [name & args] {(keyword name) args})
:option_param (fn [args] args)
:numlist_option (fn [name & args] {(keyword name) args})
:analysis (fn [& args] {"$type" "Analysis"
:Command "analysis"
:message (str "Ignoring \"" (-> args first trim) "\"")})
:invalid (fn [& args] {"$type" "Invalid"
:Command "invalid"
:Message (str "Invalid/unexpected syntax for \"" (-> args first trim) "\"")
:Messages [{:Severity "Error"
:Messages [{:Severity "Error"
:MessageText (str "Invalid syntax for " (first args))}]})
:unsupported (fn [& args] {"$type" "Unsupported"
:unsupported (fn [& args] {"$type" "Unsupported"
:Command "unsupported"})
:unrecognized (fn [& args] (first args))
:generate (fn [& [varname expr condition]]
......@@ -84,7 +85,7 @@
:Labels (map (fn [lab] {:Value (first lab) :Label (second lab)}) labels)}
(not labels) (assoc
:Messages
[{:Severity "warning"
[{:Severity "warning"
:MessageText (str "Undefined label list " codelistname)}])
)
)
......@@ -262,7 +263,8 @@
:assignment (fn [& args] {:target (first args) :source (second args)})
:weight (fn [& args]
(when (first args)
{:wtype (let [wtype (first args)]
{:Expression (second args)
:Type (let [wtype (first args)]
(cond
(or (starts-with? wtype "fw") (starts-with? wtype "freq")) "fw"
(or (starts-with? wtype "aw") (starts-with? wtype "cell")) "aw"
......@@ -270,8 +272,7 @@
(starts-with? wtype "iw") "iw"
:default "default"
)
)
:expression (second args)}))
)}))
:append (fn [& args]
(let [files (first args)
options (second args)
......@@ -294,9 +295,11 @@
:filelist (fn [& args] args)
:varlist (fn [& args] args)
:numlist (fn [& args]
{"$type" "ValueListExpression"
:Values (map numeric-constant-expression args)}
)
(let [values (map numeric-constant-expression args)]
(if (> (count values) 1)
{"$type" "ValueListExpression"
:Values values}
(first values))))
:simple_range (fn [& [from to]]
(number-range-expression from to 1))
:step_range (fn [& [from step to]]
......@@ -306,11 +309,19 @@
:varRange (fn [& args] {:first (first args) :last (second args)})
:rules (fn [& args] args)
:simple_rules (fn [& args] args)
:elementlist (fn [& args] args)
:elementlist (fn [& args]
(let [values (map transform-recode-element args)]
(if (> (count values) 1)
{"$type" "ValueListExpression"
:Values values}
(first values))))
:nonmissing_values (fn [& _] {"$type" "UnhandledNonmissingValuesExpression"})
:missing_values (fn [& _] {"$type" "UnhandledMissingValuesExpression"})
:other_values (fn [& _] {"$type" "UnhandledValuesExpression"})
:element_range (fn [& args] {:first (first args) :last (second args)})
:element_range (fn [& args] (number-range-expression
(transform-recode-element (first args))
(transform-recode-element (second args))
1))
:rule transform-recode-rule
:simple_rule transform-recode-rule
:egen_varlist egen-varlist
......
This diff is collapsed.
(ns ^{
:author "Ole Voldsæter"
:doc "Stata functions are handled here"
} stata2sdtl.functions)
} stata2sdtl.functions
(:require
[stata2sdtl.utils :refer :all]))
(def function-mapping
{"abs" "absolute_value"
......@@ -33,13 +35,13 @@
(defn function-lookup [& args]
(let [canonical (get function-mapping (first args))]
{"$type" "FunctionCallExpression"
:Function (or canonical (first args))
:Arguments (map-indexed
(fn [i arg] {:ArgumentName (str "EXP" (inc i))
:ArgumentValue arg})
(rest args)
)
:IsSdtlName (some? canonical)}
{"$type" "FunctionCallExpression"
:Function (or canonical (first args))
:Arguments (map-indexed
(fn [i arg] {:ArgumentName (str "EXP" (inc i))
:ArgumentValue arg})
(rest args)
)
:IsSdtlName (some? canonical)}
)
)
\ No newline at end of file
(ns ^{
:author "Ole Voldsæter"
:author "Ole Voldsæter, Sigbjørn Revheim"
:doc "Functions are put here to limit complexity in the transformation map in stata2sdtl.core"
} stata2sdtl.transform-helpers
(:require
......@@ -41,67 +41,28 @@
(filter (comp not nil?) (map #(:target %) variables))
)
(defn transform-recode-element
[element]
(cond
(= element "min") [{"$type" "NumericMinimumValueExpression"}]
(= element "max") [{"$type" "NumericMaximumValueExpression"}]
(= element "missing") [{"$type" "UnhandledMissingValuesExpression"}]
(map? element) element
:default (numeric-constant-expression element)))
(defn transform-recode-rule
[& args]
(m/filter-vals
some?
{:FromValue (cond
(= (first (first args)) "min") [{"$type" "NumericMinimumValueExpression"}]
(= (first (first args)) "max") [{"$type" "NumericMaximumValueExpression"}]
(= (first (first args)) "missing") [{"$type" "UnhandledMissingValuesExpression"}]
(= (first (first args)) "nonmissing") [{"$type" "UnhandledNonmissingValuesExpression"}]
(= (first (first args)) "else") [{"$type" "UnhandledValuesExpression"}]
:default args)
:To (second args)
{"$type" "RecodeRule"
:FromValue (cond
(= (first args) "missing") [{"$type" "UnhandledMissingValuesExpression"}]
(= (first args) "nonmissing") [{"$type" "UnhandledNonmissingValuesExpression"}]
(= (first args) "else") [{"$type" "UnhandledValuesExpression"}]
:default (first args))
:To (transform-recode-element (second args))
:Label (if (> (count args) 2) (nth args 2))}))
(defn transform-recode-rule2
[& args]
(m/filter-vals
some?
{:FromValue (cond
(= (first (first args)) "min") [{"$type" "NumericMinimumValueExpression"}]
(= (first (first args)) "max") [{"$type" "NumericMaximumValueExpression"}]
:default (let [values (filter number? (first args))
ranges (filter map? (first args))]
(cond-> {:args args :fTo (second args), :Label (if (> (count args) 2) (nth args 2))}
(not-empty values) (assoc :FromValue values)
(not-empty ranges) (assoc :fromValueRange ranges)
)
))
:To (second args)
:Label (if (> (count args) 2) (nth args 2))}))
(defn transform-recode-rule_test
[& args]
(m/filter-vals
some?
{:specialFromValue (first args)
:to (second args)
:label (if (> (count args) 2) (nth args 2))})
)
(defn transform-recode-rule_original
[& args]
(m/filter-vals
some?
(if (string? (first args))
{:specialFromValue (first args)
:to (second args)
:label (if (> (count args) 2) (nth args 2))}
;; else
(let [values (filter number? (first args))
ranges (filter map? (first args))]
(cond-> {:to (second args), :label (if (> (count args) 2) (nth args 2))}
(not-empty values) (assoc :fromValue values)
(not-empty ranges) (assoc :fromValueRange ranges)
)
)
))
)
(defn make-seq-comparable [s]
(let [digit-set #{\0 \1 \2 \3 \4 \5 \6 \7 \8 \9}]
{:content s
......@@ -201,7 +162,7 @@
:Command "collapse"
:GroupByVariables (map variable-symbol-expression by-vars)
:AggregateVariables (reduce #(concat (make-compute-list %1) (make-compute-list %2)) aggregations)
:WeightVariable (variable-symbol-expression weight)
:WeightVariable weight
:Condition filter
:Cellwise (contains? options :cw)
:mod-varlist (concat
......
(ns ^{
:author "Ole Voldsæter"
:author "Ole Voldsæter, Sigbjørn Revheim"
:doc "Misc. functions and dynamic variables"
} stata2sdtl.utils
(:require
[clj-diff.core :refer [diff]])
[clj-diff.core :refer [diff]]
[clojure.pprint :as pp])
)
(def ^:dynamic *variables* [])
......@@ -26,6 +27,21 @@
{"$type" "StringConstantExpression"
:Value value})
(defn get-numeric-constant
[value]
(if (map? value)
(if (= (get value "$type") "NumericConstantExpression")
value
nil)
nil))
(defn print-and-return
[label x]
(do
(pp/pprint label)
(pp/pprint x)
x))
(defn get-var-length-option [k l m]
(loop [opts (keys m)]
(let [looking-for (apply str (take (inc l) (str k)))
......
......@@ -224,44 +224,41 @@
:Command "compute"
:Variable {"$type" "VariableSymbolExpression"
:VariableName "a"}
:Expression {"$type" "functionCallExpression"
:Arguments [{"$type" "VariableSymbolExpression"
:VariableName "a"}]
:Expression {"$type" "FunctionCallExpression"
:Function "agg_mean"
:IsSdtlName true}}
:IsSdtlName true
:Arguments [{"$type" "VariableSymbolExpression"
:VariableName "a"}]}}
{"$type" "Compute"
:Command "compute"
:Variable {"$type" "VariableSymbolExpression"
:VariableName "b"}
:Expression {"$type" "functionCallExpression"
:Arguments [{"$type" "VariableSymbolExpression"
:VariableName "c"}]
:Expression {"$type" "FunctionCallExpression"
:Function "agg_mean"
:IsSdtlName true}}
:IsSdtlName true
:Arguments [{"$type" "VariableSymbolExpression"
:VariableName "c"}]}}
{"$type" "Compute"
:Command "compute"
:Variable {"$type" "VariableSymbolExpression"
:VariableName "d"}
:Expression {"$type" "functionCallExpression"
:Arguments [{"$type" "VariableSymbolExpression"
:VariableName "d"}]
:Expression {"$type" "FunctionCallExpression"
:Function "agg_max"
:IsSdtlName true}})
:WeightVariable {:expression {"$type" "VariableSymbolExpression"
:variableName "f"}
:wtype "fw"}
:Cellwise true
:IsSdtlName true
:Arguments [{"$type" "VariableSymbolExpression"
:VariableName "d"}]}})
:WeightVariable {:Expression {"$type" "VariableSymbolExpression"
:VariableName "f"}
:Type "fw"}
:Condition nil
:mod-varlist ("e"
"a"
"b"
"d")}
:Cellwise true
:mod-varlist ("e" "a" "b" "d")}
)
"Should return a collapse object. Variable 'e' must be first in :mod-varlist"
)
(is
(=
(parse-and-transform "append using a, test_option(1 2/4 5 (2) 9 10 13 : 19, %1.2)")
(parse-and-transform "append using a, test_option(1, 2/4 5 (2) 9, 10 13 : 19)")
'{"$type" "AppendDatasets"
:Command "appendDatasets"
:FileName ("a")
......@@ -279,8 +276,7 @@
{"$type" "NumberRangeExpression"
:NumberRangeStart 10
:NumberRangeEnd 19
:NumberRangeIncrement 3})}
"%1.2")}
:NumberRangeIncrement 3})})}
:mod-varlist ()}
)
)
......@@ -417,8 +413,8 @@
:Condition nil
:Expression {"$type" "NumericConstantExpression"
:Value 1}
:SourceInformation {:lineNumberStart 0
:originalSourceText "gen b=1"}}]
:SourceInformation {:LineNumberStart 0
:OriginalSourceText "gen b=1"}}]
:stack ()
:variables ["a"
"b"]}
......
(ns stata2sdtl.transform-helpers-test
(:require [clojure.test :refer :all]
[stata2sdtl.transform-helpers :refer :all]
[stata2sdtl.core :refer :all])
)
[stata2sdtl.core :refer :all]))
(deftest recode-varlist-test
(is
......@@ -11,57 +11,43 @@
:target "a2"}
{:source "b"
:target "b2"}])
"Should return a2 and b2 as targets"
)
"Should return a2 and b2 as targets")
(is
(= (recode-varlist ["a" "b"] {:pre ["nu_"]})
[{:source "a"
:target "nu_a"}
{:source "b"
:target "nu_b"}])
"Should return a2 and b2 as targets"
)
)
"Should return a2 and b2 as targets"))
(deftest extract-target-variables-test
(is
(= (extract-target-variables ["a" "b" {:source "foo" :target "bar"}])
["bar"]
)
"Should return [\"bar\"]"
)
)
["bar"])
(deftest transform-recode-rule-test
(is
(= (transform-recode-rule [{:first 1, :last 4} 9] 7 "foo")
{:FromValue [9]
:fromValueRange [{:first 1
:last 4}]
:Label "foo"
:To 7})
)
)
"Should return [\"bar\"]"))
(deftest make-seq-comparable-test
(is
(= (make-seq-comparable "foo123bar")
{:alpha-part "foo"
:content "foo123bar"
:num-part 123}
)
"Should return \"foo\" and 123"
)
:num-part 123})
"Should return \"foo\" and 123")
(is
(= (order-seq-comparator (make-seq-comparable "aa1") (make-seq-comparable "aa2"))
-1
)
"Should return -1"
)
-1)
"Should return -1")
(is
(= (order-seq-comparator (make-seq-comparable "aa1") (make-seq-comparable "ab1"))
-1
)
"Should return -1"
)
)
\ No newline at end of file
-1)
"Should return -1"))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment