Extras - selected functionalities of selected packages¶
In [1]:
using DataFrames
In [97]:
using Pipe
In [2]:
using FreqTables
In [3]:
using CategoricalArrays
In [4]:
using BenchmarkTools
In [5]:
ENV["LINES"] = 15
Out[5]:
In [6]:
df = DataFrame(a=rand('a':'d',1000), b=rand(["x","y","z"],1000))
# observe that dimensions are sorted if possible
ft = freqtable(df, :a, :b)
Out[6]:
In [7]:
# you can index the result using number of names
ft[1,1], ft['b',"z"]
Out[7]:
In [8]:
# getting proportions - 1 means we want to calculate them in rows
# (first dimension)
pr = prop(ft, margins=1)
Out[8]:
In [9]:
pr[1,:] |> sum,pr[4,:] |> sum
Out[9]:
In [10]:
# and columns are normalized to 1.0 now
pc = prop(ft, margins=2)
Out[10]:
In [11]:
pc[:,"x"] |> sum,pc[:,"y"] |> sum
Out[11]:
In [12]:
p = prop(ft)
Out[12]:
In [13]:
p |> sum
Out[13]:
In [14]:
x = categorical(rand(1:3,10))
Out[14]:
In [15]:
# rerodering levels and adding an extra level
levels!(x,[3,1,2,4])
# order is preserved and not-used level is shown
freqtable(x)
Out[15]:
In [16]:
# by default missings are listed
freqtable([1,1,2,3,missing])
Out[16]:
In [17]:
# but we can skip them
freqtable([1,1,2,3,missing],skipmissing=true)
Out[17]:
In [18]:
df = DataFrame(a=rand(3:4,1_000),b=rand(5:6,1_000))
# now dimensions are numbers
ft = freqtable(df, :a, :b)
Out[18]:
In [19]:
# this is an error - standard array indexing takes precedence
ft[3,5]
In [20]:
# you have to use Name() wrapper
ft[FreqTables.Name(3),FreqTables.Name(5)]
Out[20]:
DataFramesMeta.jl - working on DataFrame¶
DataFramesMeta.jl provides a more terse syntax due to the benefits of metaprogramming.
In [21]:
using DataFramesMeta
In [22]:
df = DataFrame(x=1:8, y='a':'h',z=repeat([true,false],outer=4))
Out[22]:
In [23]:
# expressions with columns of DataFrame
@btime DataFramesMeta.@with($df, :x + :z)
Out[23]:
In [24]:
@btime +($df.x,df.z)
Out[24]:
In [25]:
# you can define code blocks
@with df begin
a = :x[:z] # 1,3,5,7
b = :x[.!:z] # 2,4,6,8
:y + [a;b] # [a;b] : 1,3,5,7,2,4,8
end
Out[25]:
In [26]:
# @with creates hard scope so variables do not leak out
a
In [27]:
df2 = DataFrame(a=[:a,:b,:c])
Out[27]:
In [28]:
# sometimes we want to work on a raw Symbol, ^() escapes it
@with(df2, :a .== ^(:a))
Out[28]:
In [29]:
@with(df2, :a .== Symbol("a"))
Out[29]:
In [30]:
x_str = "x"
y_str = "y"
df2 = DataFrame(x=1:3, y=4:6, z=7:9)
Out[30]:
In [31]:
# cols(expression) selects given columns
@with(df2,cols(x_str) + cols(y_str))
Out[31]:
In [32]:
@with(df2,$(x_str) + $(y_str))
Out[32]:
In [33]:
df
Out[33]:
In [34]:
# a very useful macro for filtering
DataFramesMeta.@subset(df, :x .< 4, :z .== true)
Out[34]:
In [35]:
# create a new DataFrame based on the old one
DataFramesMeta.@select(df, :x, y = 2*:x,z=:y)
Out[35]:
In [36]:
# create a new DataFrame adding columns based on old one
@btime DataFramesMeta.@transform($df,:x = 2*:x, :y=:x)
Out[36]:
In [37]:
@btime transform($df,:x=>ByRow(x->2*x)=>:x,:x=>:y)
Out[37]:
In [38]:
# dorting into a new data frame, less powerful than sort,
# but lightweight
@btime DataFramesMeta.@orderby($df,:z,-:x)
Out[38]:
In [39]:
@btime sort($df,[:z,:x],rev=[false,true])
Out[39]:
In [40]:
using Chain
In [41]:
# chainning of operations on DataFrame
@chain df begin
@subset(:x .< 5)
@orderby(:z)
@transform(:x²= :x .^2)
@select(:z,:x,:x²)
end
Out[41]:
DataFramesMeta -working on grouped DataFrame¶
In [42]:
df = DataFrame(a=1:12, b=repeat('a':'d', outer=3))
Out[42]:
In [43]:
g = groupby(df, :b)
Out[43]:
In [44]:
using Statistics
In [45]:
# groupby + combine in one shot
@btime DataFramesMeta.@by(df,:b,:first=first(:a),:last=last(:a),:mean=mean(:a))
Out[45]:
In [46]:
@btime @chain df begin
groupby(:b)
combine(:a=>first=>:first,:a=>last=>:last,:a=>mean=>:mean)
end
Out[46]:
In [51]:
# the same as by but on grouped DataFrame
@btime @combine($g,:first=first(:a),:last=last(:a), :mean=mean(:a))
Out[51]:
In [52]:
# similar in DataFrames.jl
@btime combine($g,:a .=> [first,last, mean] .=> [:first,:last,:mean])
Out[52]:
In [114]:
# perform operations within a group and return ungrouped DataFrame
@btime @transform($g, :center=mean(:a), :centered = :a .- mean(:a))
Out[114]:
In [115]:
@btime transform($g,:a.=>[mean,(a-> (a .- mean(a)))] .=> [:center, :centered])
Out[115]:
In [110]:
# this is defined in DataFrames.jl
DataFrame(g)
Out[110]:
In [111]:
# actually this is not the same as DataFrame()
# as it preserves the original row order
@transform(g)
Out[111]:
DataFramesMeta - rowwise operations on DataFrame¶
In [320]:
df = DataFrame(a=1:12, b=repeat(1:4, outer=3))
Out[320]:
In [321]:
# such conditions are often needed but are complex to write
@btime @transform($df, :x=ifelse.((:a .> 6) .& (:b .== 4),"yes","no"))
Out[321]:
In [322]:
@btime transform($df, [:a,:b]=>
ByRow((a,b)->ifelse((a > 6) & (b == 4),"yes","no"))=>:x)
Out[322]:
In [323]:
# one option is to use a function that works on a single observation and
# broadcast it
myfunc(a,b) = a > 6 && b == 4 ? "yes" : "no"
@btime @transform($df, :x=myfunc.(:a,:b))
Out[323]:
In [324]:
# or you can use @eachrow macro that allows you to process DataFrame rowwise
@btime @eachrow $df begin
@newcol x::Vector{String}
:x = :a > 6 && :b == 4 ? "yes" : "no"
end
Out[324]:
Very Fast!!!
In [325]:
@btime @chain $df begin
_.x = ifelse.((_.a .> 6) .& (_.b .== 4) ,"yes","no")
end
df
Out[325]:
In [329]:
@btime transform($df,[:a,:b]=>ByRow((a,b)->ifelse(a > 6 && b == 4,"yes","no"))=>:x)
Out[329]:
You can also use eachrow from DataFrames to perform the same transformation.
Very Fast!!!
In [330]:
df2 = copy(df)
df2.x = Vector{String}(undef,nrow(df2))
@btime begin
for row in eachrow($df2)
row[:x] = row[:a] > 6 && row[:b] == 4 ? "yes" : "no"
end
end
df2
Out[330]:
Visualizing data with StatsPlots¶
In [331]:
using StatsPlots
default(fmt=:png)
We present only a minimal functionality of the package
In [332]:
using Random
Random.seed!(1)
df = DataFrame(x=sort(randn(1_000)), y=randn(1_000),
z=[fill("b",500);fill("a",500)])
Out[332]:
In [333]:
@df df plot(:x,:y, legend=:topleft,label="y(x)")
Out[333]:
In [334]:
@df df density(:x,label="")
Out[334]:
In [335]:
@df df histogram(:y, label="y")
Out[335]:
In [336]:
@df df boxplot(:z,:x,label="x")
Out[336]:
In [337]:
@df df violin(:z,:y,label="y")
Out[337]:
In [ ]: