Commit 20a9e83c authored by Ole Voldsæter's avatar Ole Voldsæter
Browse files

Merge branch '50-numberlistexpression' into 'develop'

Resolve "Implement NumberListExpression"

Closes #50

See merge request c2metadata/stata-sdtl-converter!20
parents ca2f00c2 c05447f6
Pipeline #14589 passed with stages
in 2 minutes and 9 seconds
......@@ -137,7 +137,7 @@ format_vars
;
<display_format>
: #"\%[^\s]+"
: #"\%[^\s,)]+"
;
drop_cases
......@@ -320,6 +320,7 @@ option_param
: varlist
| numlist
| string
| display_format
;
local
......
......@@ -30,8 +30,8 @@
;; mapping of grammar production names to transformation functions
(def transformations
{
:options (fn [& args] (reduce #(assoc %1 (keyword (first %2)) (second %2)) {} args))
:option (fn [& args] args)
:options (fn [& args] (reduce #(assoc %1 (keyword (first %2)) (apply concat (second %2))) {} args))
:option (fn [name & args] [name args])
:option_param (fn [& args] args)
:analysis (fn [& args] {:command "analysis"
:message (str "Ignoring \"" (-> args first trim) "\"")})
......@@ -259,7 +259,28 @@
}))
:filelist (fn [& args] args)
:varlist (fn [& args] args)
:numlist (fn [& args] args)
:numlist (fn [& args]
{"$type" "NumberListExpression"
:values args}
)
:simple_range (fn [& [from to]]
{"$type" "NumberRangeExpression"
:from from
:to to
:step 1}
)
:step_range (fn [& [from step to]]
{"$type" "NumberRangeExpression"
:from from
:to to
:step step}
)
:diff_step_range (fn [& [from second to]]
{"$type" "NumberRangeExpression"
:from from
:to to
:step (- second from)}
)
:varRange (fn [& args] {:first (first args) :last (second args)})
:rules (fn [& args] args)
:simple_rules (fn [& args] args)
......
......@@ -6,15 +6,13 @@
)
; -------- compute ? ---------------
; anycount
; anymatch
; anyvalue
; concat
; cut
; ends
; group
(defonce compute-funcs {
(def compute-funcs {
:anycount "anycount"
:anymatch "anymatch"
:anyvalue "anyvalue"
:diff "diff"
:rowfirst "rowfirst"
:rowlast "rowlast"
......@@ -29,9 +27,13 @@
:rowtotal "rowtotal"
:std "std_val"
:tag "tag"
:concat "concat"
:cut "cut"
:group "group"
:ends "ends"
})
(defonce aggregate-funcs {
(def aggregate-funcs {
:count "agg_count"
:iqr "agg_iqr"
:kurt "agg_kurt"
......@@ -58,7 +60,7 @@
{"$type" "VariableListExpression" :variables (map (fn [varname] {:variableName varname , "$type" "VariableSymbolExpression"}) (expand-varlist varlist *variables*))}
)
(defn make-compute [varname funcname args filter]
(defn make-compute [varname funcname args _filter]
{:command "compute"
:variable varname
:expression {:function funcname
......@@ -69,106 +71,154 @@
args
)
"$type" "FunctionCallExpression"}
:condition filter
:condition _filter
:mod-varlist (vec (conj *variables* varname))}
)
(defn make-aggregate [by varname funcname expr filter]
(defn make-any [varname funcname varlist options _filter]
(make-compute
varname
funcname
[(make-varlist-expression varlist)
{"$type" "NumberListExpression" :values (->> (get-var-length-option :values 1 options) (get options) first)}]
_filter
)
)
(defn make-aggregate [by varname funcname expr _filter]
{:command "aggregate"
:groupByVariables by
:aggregateVariables [(dissoc (make-compute varname funcname expr filter) :mod-varlist)]
:aggregateVariables [(dissoc (make-compute varname funcname expr _filter) :mod-varlist)]
:mod-varlist (vec (conj *variables* varname))}
)
(defn varlist-compute [varname funcname varlist filter options]
(defn varlist-compute [varname funcname varlist _filter options]
(case funcname
:rowpctile (make-compute
varname
"rowpctile"
(let [p (:p options [[50]])]
(let [p (:p options {:values [50]})]
[(make-varlist-expression varlist)
{"$type" "NumericConstantExpression" :value (-> p first first)}]
{"$type" "NumericConstantExpression" :value (-> p :values first)}]
)
filter)
_filter)
:rowtotal (make-compute
varname
"rowtotal"
[(make-varlist-expression varlist)]
filter)
[(make-varlist-expression varlist) (-> (get-var-length-option :missing 1 options) nil? not)]
_filter)
:tag (make-compute
varname
"tag"
[(make-varlist-expression varlist)]
filter)
[(make-varlist-expression varlist) (-> (get-var-length-option :missing 1 options) nil? not)]
_filter)
:concat (make-compute
varname
"concat"
[(make-varlist-expression varlist)
(->> (get-var-length-option :format 1 options) (get options) first)
(->> (get-var-length-option :decode 1 options) nil? not)
(->> (get-var-length-option :maxlength 4 options) (get options) first :values first)
(->> (get-var-length-option :punct 1 options) (get options) first)]
_filter)
:cut (make-compute
varname
"cut"
[(first varlist)
(->> options :at (transduce (map #(:values %)) concat) (assoc {"$type" "NumberListExpression"} :values))
(->> (get-var-length-option :group 1 options) (get options) first :values first)
(->> (get-var-length-option :icodes 2 options) nil? not)
(->> (get-var-length-option :label 3 options) nil? not)]
_filter)
:anycount (make-any varname funcname varlist options _filter)
:anymatch (make-any varname funcname varlist options _filter)
:anyvalue (make-any varname funcname varlist options _filter)
; default
(make-compute
varname
(get compute-funcs (keyword funcname))
[(make-varlist-expression varlist)]
filter
_filter
)
)
)
(defn varlist-aggregate [by-clause varname funcname varlist filter options]
(defn varlist-aggregate [by-clause varname funcname varlist _filter options]
(case funcname
:mode (unsupported-func funcname)
(make-aggregate
by-clause varname
(get aggregate-funcs (keyword funcname))
[(make-varlist-expression varlist)]
filter)
_filter)
)
)
(defn egen-varlist [& [by-clause varname funcname varlist filter options]]
(defn egen-varlist [& [by-clause varname funcname varlist _filter options]]
(let [funcname (keyword funcname)]
(cond
(some #{funcname} (keys compute-funcs)) (varlist-compute varname funcname varlist filter options)
(some #{funcname} (keys aggregate-funcs)) (varlist-aggregate by-clause varname funcname varlist filter options)
(some #{funcname} (keys compute-funcs)) (varlist-compute varname funcname varlist _filter options)
(some #{funcname} (keys aggregate-funcs)) (varlist-aggregate by-clause varname funcname varlist _filter options)
:default (unsupported-func funcname)
)
)
)
(defn egen-fill [& [varname numlist filter]]
(unsupported-func "fill")
(defn egen-fill [& [varname numlist _filter]]
(make-compute varname "fill" [numlist] _filter)
)
(defn egen-mtr [& [varname exp1 exp2 filter]]
(defn egen-mtr [& [varname exp1 exp2 _filter]]
(make-compute
varname
"mtr"
[exp1 exp2]
filter)
_filter)
)
(defn egen-expr [& [by-clause varname funcname expr filter options]]
(defn egen-expr [& [by-clause varname funcname expr _filter options]]
(let [funcname (keyword funcname)]
(case funcname
:std (make-compute
varname
"std_val"
(let [m (get-var-length-option :mean 1 options)
s (get-var-length-option :std 1 options)
m (if m (get options m) [[0]])
s (if s (get options s) [[1]])]
[expr
{"$type" "NumericConstantExpression" :value (-> m first first)}
{"$type" "NumericConstantExpression" :value (-> s first first)}]
)
filter)
:std (make-compute
varname
"std_val"
(let [m (get-var-length-option :mean 1 options)
s (get-var-length-option :std 1 options)
m (if m (get options m) {:values [0]})
s (if s (get options s) {:values [1]})]
[expr
{"$type" "NumericConstantExpression" :value (-> m :values first)}
{"$type" "NumericConstantExpression" :value (-> s :values first)}]
)
_filter)
:pctile (make-aggregate
by-clause
varname
(get aggregate-funcs (keyword funcname))
[expr
{"$type" "NumericConstantExpression" :value (-> (:p options [[50]]) first first)}]
filter)
;;;----- weird ones -------------
:rank (unsupported-func funcname)
:pc (unsupported-func funcname)
{"$type" "NumericConstantExpression" :value (-> (:p options {:values [50]}) :values first)}]
_filter)
:pc (make-aggregate
by-clause
varname
(get aggregate-funcs (keyword funcname))
[expr
(-> options :prop nil? not)]
_filter)
:rank (make-aggregate
by-clause
varname
(get aggregate-funcs (keyword funcname))
[expr
{"$type" "StringConstantExpression"
:value (cond
(get-var-length-option :field 1 options) "field"
(get-var-length-option :track 1 options) "track"
(get-var-length-option :unique 1 options) "unique"
:else "default"
)}]
_filter)
;;; default
(make-aggregate
......@@ -176,7 +226,7 @@
varname
(get aggregate-funcs (keyword funcname))
[expr]
filter)
_filter)
)
)
)
\ No newline at end of file
......@@ -234,6 +234,29 @@
)
"Should return a collapse object. Variable 'e' must be first in :mod-varlist"
)
(is
(=
(parse-and-transform "append using a, test_option(1 2/4 5 (2) 9 10 13 : 19, %1.2)")
'{:command "append"
:files ("a")
:mod-varlist ()
:options {:test_option ({"$type" "NumberListExpression"
:values (1
{"$type" "NumberRangeExpression"
:from 2
:step 1
:to 4}
{"$type" "NumberRangeExpression"
:from 5
:step 2
:to 9}
{"$type" "NumberRangeExpression"
:from 10
:step 3
:to 19})}
"%1.2")}}
)
)
)
(deftest expand-macros-test
......
......@@ -58,7 +58,7 @@
:variable "some-var"})
)
(is
(= (varlist-compute "some-var" :rowpctile ["a" "b"] 5 {:p [[10]]})
(= (varlist-compute "some-var" :rowpctile ["a" "b"] 5 {:p {:values '(10)}})
'{:command "compute"
:condition 5
:expression {"$type" "FunctionCallExpression"
......@@ -86,7 +86,9 @@
:variables ({"$type" "VariableSymbolExpression"
:variableName "a"}
{"$type" "VariableSymbolExpression"
:variableName "b"})}})
:variableName "b"})}}
{:argumentName "EXP2"
:argumentValue false})
:function "rowtotal"
:isSdtlName true}
:mod-varlist ["some-var"]
......@@ -102,7 +104,9 @@
:variables ({"$type" "VariableSymbolExpression"
:variableName "a"}
{"$type" "VariableSymbolExpression"
:variableName "b"})}})
:variableName "b"})}}
{:argumentName "EXP2"
:argumentValue true})
:function "rowtotal"
:isSdtlName true}
:mod-varlist ["some-var"]
......@@ -118,7 +122,9 @@
:variables ({"$type" "VariableSymbolExpression"
:variableName "a"}
{"$type" "VariableSymbolExpression"
:variableName "b"})}})
:variableName "b"})}}
{:argumentName "EXP2"
:argumentValue false})
:function "tag"
:isSdtlName true}
:mod-varlist ["some-var"]
......@@ -134,7 +140,9 @@
:variables ({"$type" "VariableSymbolExpression"
:variableName "a"}
{"$type" "VariableSymbolExpression"
:variableName "b"})}})
:variableName "b"})}}
{:argumentName "EXP2"
:argumentValue true})
:function "tag"
:isSdtlName true}
:mod-varlist ["some-var"]
......@@ -179,7 +187,7 @@
)
)
(is
(= (egen-expr nil "somevar" "std" 5 1 {:mean [[7]]})
(= (egen-expr nil "somevar" "std" 5 1 {:mean {:values '(7)}})
'{:command "compute"
:condition 1
:expression {"$type" "FunctionCallExpression"
......@@ -198,7 +206,7 @@
)
)
(is
(= (egen-expr nil "somevar" "std" 5 1 {:std [[2]] :mean [[7]]})
(= (egen-expr nil "somevar" "std" 5 1 {:std {:values '(2)} :mean {:values '(7)}})
'{:command "compute"
:condition 1
:expression {"$type" "FunctionCallExpression"
......@@ -236,7 +244,7 @@
)
)
(is
(= (egen-expr ["a" "b"] "somevar" "pctile" 5 1 {:p [[10]]})
(= (egen-expr ["a" "b"] "somevar" "pctile" 5 1 {:p {:values '(10)}})
'{:aggregateVariables [{:command "compute"
:condition 1
:expression {"$type" "FunctionCallExpression"
......@@ -254,4 +262,23 @@
:mod-varlist ["somevar"]}
)
)
)
(deftest egen-fill-test
(is
(=
(egen-fill "somevar" {:values '(1 2 3)} nil)
'{:command "compute"
:condition nil
:expression {"$type" "FunctionCallExpression"
:arguments ({:argumentName "EXP1"
:argumentValue {:values (1
2
3)}})
:function "fill"
:isSdtlName true}
:mod-varlist ["somevar"]
:variable "somevar"}
)
)
)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment