Data Science & Analysis

183 commands

datamash --header-in sum 4 < data.tsv datamash -t, --header-in sum 4 < data.csv datamash -t, --header-in mean 2 median 2 pstdev 2 < data.csv datamash -t, --header-in -g 3 mean 4 < data.csv datamash -t, --header-in perc:95 4 < data.csv datamash -t, --header-in -s -g 3 mean 4 < data.csv datamash -H -t, --header-in sum 4 < data.csv datamash -t, --header-in q1 2 q3 2 iqr 2 < data.csv datamash -t, --header-in unique 3 countunique 3 < data.csv datamash -t, --header-in -g 3 collapse 1 < data.csv datamash -t, --header-in geomean 2 harmmean 2 < data.csv datamash -t, --header-in trimmean:0.1 2 < data.csv datamash -t, --header-in pstdev 2 pvar 2 svar 2 mad 2 < data.csv datamash -t, --header-in spearson 2:4 < data.csv datamash -t, --header-in pcov 2:4 < data.csv datamash -t, --header-in range 2 absmax 2 < data.csv datamash -t, --header-in -g 3 count 4 sum 4 min 4 max 4 < data.csv datamash -t, --header-in -g 3 trimmean:0.1 4 < data.csv datamash -t, transpose < data.csv datamash -t, check < data.csv datamash --no-strict -t, sum 1 < ragged.csv mlr --csv head -n 5 data.csv mlr --csv stats1 -a mean,min,max -f score data.csv mlr --csv stats1 -a sum,mean -f salary -g dept data.csv mlr --csv sort -f age data.csv mlr --csv sort -nr score data.csv mlr --csv sort -f dept -nr salary data.csv mlr --csv filter '$age > 30' data.csv mlr --csv cut -f name,age data.csv mlr --csv put '$ratio = $salary / $score' data.csv mlr --csv put -S '$pct = $score * 100' data.csv mlr --csv put '$ts = sec2gmt(1700000000)' data.csv mlr --icsv --ojson cat data.csv mlr --icsv --ojsonl cat data.csv mlr --json head -n 1 data.json mlr --json tail -n 2 data.json mlr --csv join -f emp-ids.csv -j name data.csv mlr --csv nest --explode --values --across-records -f tags tags.csv mlr --csv uniq -a data.csv mlr --csv uniq -g dept data.csv mlr --csv count-distinct -f dept data.csv mlr --csv count-distinct -n -f dept data.csv mlr --csv top -n 5 -f score data.csv mlr --csv rename name,employee data.csv mlr --csv tac data.csv mlr --csv sample -k 2 data.csv mlr --csv reorder -f name,score data.csv mlr --csv merge-fields -a sum -f salary,score -o total data.csv mlr --csv histogram -f score --lo 0.7 --hi 1.0 --nbins 3 data.csv mlr --csv step -a delta -f score data.csv printf 'a,b,c\n1,,3\n' | mlr --csv remove-empty-columns xsv headers data.csv xsv stats data.csv xsv stats --median --mode data.csv xsv stats -s age,salary data.csv xsv select name,age data.csv xsv search eng data.csv xsv frequency -s dept data.csv xsv table data.csv xsv slice -s 2 -e 4 data.csv xsv sort -s age data.csv xsv join name data.csv name emp-ids.csv xsv count data.csv xsv sample 3 data.csv xsv flatten data.csv xsv fmt -t '|'xsv cat rows a.csv b.csv xsv cat columns a.csv b.csv xsv input -d '|'xsv index data.csv xsv split --size 2 split data.csv xsv partition dept part data.csv xsv fixlengths data.csv csvcut -n data.csv csvcut -c name,age data.csv csvstat data.csv csvstat --mean data.csv csvstat --median data.csv csvstat --json data.csv csvgrep -c name -m 'alice' data.csv csvgrep -c name -r '^a' data.csv csvgrep -i -c dept -m 'ENG' data.csv csvjoin --left -c name data.csv emp-ids.csv csvsort -c age data.csv csvsort -c age -r data.csv csvstack a.csv b.csv csvstack -g one,two a.csv b.csv csvlook data.csv csvformat -T data.csv csvformat -D '|' data.csv csvpy data.csv csvsql --query 'SELECT dept, AVG(salary) FROM data GROUP BY dept' data.csv csvsql --query 'SELECT dept, MAX(salary) AS top FROM data GROUP BY dept' data.csv csvsql --db sqlite:///test.db --insert data.csv csvjson data.csv csvjson --indent 2 data.csv in2csv -f json api.json in2csv data.xlsx in2csv --sheet Sheet2 book.xlsx jq 'group_by(.dept) | map({dept: .[0].dept, total: map(.salary) | add})' data.json jq -r '.[] | [.name, .age] | @tsv' data.json jq 'map(select(.age > 30))' data.json jq 'sort_by(.age) | reverse | .[0]' data.json jq 'length' data.json jq -n '{sum: (100 + 200)}'jq '.[0] | keys' data.json jq --arg v alice '.[] | select(.name == $v)' data.json jq '.results[0:2] | map(.title)' data.json jq 'to_entries[] | "\(.key): \(.value)"' object.json jq -c '.[]' data.json jupyter lab --no-browser --port 8888 jupyter notebook --no-browser jupyter nbconvert --to html notebook.ipynb jupyter nbconvert --to markdown notebook.ipynb jupyter nbconvert --to script notebook.ipynb jupyter nbconvert --execute --to notebook --inplace notebook.ipynb jupyter nbconvert --clear-output --inplace notebook.ipynb jupyter nbconvert --to pdf notebook.ipynb jupyter kernelspec list jupyter kernelspec install --user ./mykernel jupyter trust notebook.ipynb ipython -c 'print(2**10)'ipython --pdb ipython --matplotlib gnuplot -e 'set terminal pngcairo; set output "out.png"; set datafile separator ","; plot "data.csv" using 2:4 with lines'gnuplot -e 'set terminal svg; set output "out.svg"; set datafile separator ","; plot "data.csv" using 2:4 with lines'gnuplot -e 'set terminal pngcairo size 1200,800; set output "big.png"; set datafile separator ","; plot "data.csv" using 2:4 with lines'gnuplot -e 'set terminal pngcairo; set output "hist.png"; set datafile separator ","; plot "data.csv" using 4 smooth frequency with boxes'gnuplot -e 'set terminal pngcairo; set output "scatter.png"; set datafile separator ","; plot "data.csv" using 2:4 with points pt 7'gnuplot -e 'set terminal pngcairo; set output "log.png"; set logscale y; set datafile separator ","; plot "data.csv" using 2:4 with lines'gnuplot -e 'set terminal pngcairo; set output "lab.png"; set title "Age vs Salary"; set xlabel "Age"; set ylabel "Salary"; set datafile separator ","; plot "data.csv" using 2:4 with lines'gnuplot -e 'set datafile separator ","; plot "data.csv" using 2:4 with lines' -p gnuplot analyze.gp gnuplot -e 'set terminal dumb size 60,20; plot sin(x)'Rscript -e 'summary(read.csv("data.csv"))'Rscript -e 'x <- c(1,2,3); print(mean(x))'Rscript -e 'df <- read.csv("data.csv"); print(tapply(df$salary, df$dept, mean))'Rscript -e 'df <- read.csv("data.csv"); png("rplot.png"); plot(df$age, df$salary); dev.off()'Rscript --vanilla -e 'print(1+1)'Rscript -e 'df <- read.csv("data.csv"); print(quantile(df$salary))'Rscript -e 'df <- read.csv("data.csv"); print(cor(df$age, df$salary))'Rscript -e 'df <- read.csv("data.csv"); print(head(df, 3))'Rscript -e 'df <- read.csv("data.csv"); print(table(df$dept))'Rscript -e 'df <- read.csv("data.csv"); print(summary(lm(salary ~ age, data = df))$coefficients)'Rscript -e 'df <- read.csv("data.csv"); png("rhist.png"); hist(df$salary, main="Salary distribution"); dev.off()'Rscript analyze.R Rscript -e 'x <- rnorm(1000, mean=50, sd=10); print(summary(x))'Rscript -e 'df <- read.csv("data.csv"); print(colMeans(df[c("age","salary","score")]))'sqlite3 data.db -header -column 'SELECT dept, COUNT(*), AVG(salary) FROM employees GROUP BY dept;'sqlite3 data.db '.mode csv' 'SELECT * FROM employees;' > employees.csv sqlite3 data.db -csv 'SELECT * FROM employees LIMIT 2;'sqlite3 :memory: 'CREATE TABLE t(x); INSERT INTO t VALUES (42); SELECT x*2 FROM t;'sqlite3 data.db 'SELECT * FROM employees ORDER BY CAST(salary AS INTEGER) DESC LIMIT 2;'sqlite3 data.db 'WITH s AS (SELECT dept, AVG(salary) AS a FROM employees GROUP BY dept) SELECT * FROM s WHERE a > 70000;'sqlite3 data.db 'SELECT dept, ROUND(AVG(salary),2) AS avg_salary FROM employees GROUP BY dept ORDER BY avg_salary DESC;'echo 'scale=4; 22/7' | bc -l echo 'sqrt(144)' | bc -l seq 1 10 | paste -sd+ | bc numfmt --to=si 1500000 numfmt --to=iec --suffix=B 1048576 paste -sd, list.txt comm -23 x1.txt x2.txt join -t, -1 1 -2 1 ids.s.csv scores.s.csv join -t, -1 1 -2 1 <(sort -t, -k1,1 ids.csv) <(sort -t, -k1,1 scores.csv)split -l 10 big.csv chunk_shuf -n 2 data.csv factor 123456789 seq 1 10 | awk '{s+=$1} END {print s}'sort -t, -k4 -nr data.csv | head -2 awk -F, 'NR>1 {s+=$4; n++} END {print s/n}' data.csv awk -F, 'NR>1 {a[$3]+=$4} END {for (k in a) print k, a[k]}' data.csv cut -d, -f3 data.csv | sort | uniq -c | sort -nr tail -n +2 data.csv | head -2 tr ',' '\t' < data.csv | head -2 paste <(cut -d, -f1 data.csv) <(cut -d, -f4 data.csv) | head -2 echo 'analysis step 1' | ts '%F %T'sort sp.txt | sponge sp.txt python3 -c 'import pandas as pd; df=pd.read_csv("data.csv"); print(df.describe())'python3 -c 'import pandas as pd; df=pd.read_csv("data.csv"); print(df.groupby("dept")["salary"].mean())'python3 -c 'import pandas as pd; df=pd.read_csv("data.csv"); print(df[df.age > 30])'python3 -c 'import pandas as pd; df=pd.read_csv("data.csv"); print(df.sort_values("salary", ascending=False).head(3))'python3 -c 'import numpy as np; print(np.mean([1,2,3,4]))'python3 -m pip install pandas numpy matplotlib

Search all 7,657 commands instead.