In [11]:
using DataFrames
using Pipe
using Chain
using CategoricalArrays
using PooledArrays
using BenchmarkTools
using Random
using LinearAlgebra
In [3]:
# row가 10줄만 표시 되게 설정
ENV["LINES"] = 10
Out[3]:
In [2]:
x = DataFrame(rand(5,1000), :auto)
Out[2]:
In [6]:
@btime $x[!,500]
@btime $x.x500;
When working with data DataFrame use barrier functions or type annotation¶
In [23]:
xx = DataFrame(rand(1_000_000,2),:auto)
yy ,zz = xx[!,1],xx[!,2]
zz
Out[23]:
In [8]:
function f_bad() # this function will be slow
Random.seed!(1)
x = DataFrame(rand(1_000_000,2), :auto)
y,z = x[!,1],x[!,2]
p = 0.0
for i in 1:nrow(x)
p += y[i]*z[i]
end
p
end
Out[8]:
In [10]:
@btime f_bad()
# if you run @code_warntype f_bad() then you notice
# that Julia does not know column types of `DataFrame`
Out[10]:
아래 그림에서 p,y,z 변수가 Any type으로 처리 되어 매우 느리게 된다.
In [24]:
@code_warntype f_bad()
solution 1 is to use barrier function (it should be possible to use it in almost any code)
In [12]:
function f_inner(y,z)
p = 0.0
for i in 1:length(y)
p += y[i]*z[i]
end
p
end
function f_barrier() # extract the work to an inner function
Random.seed!(1)
x = DataFrame(rand(1_000_000,2),:auto)
f_inner(x[!,1],x[!,2])
end
function f_inbuilt() # or use inbuilt function if possible
Random.seed!(1)
x = DataFrame(rand(1_000_000,2), :auto)
dot(x[!,1],x[!,2])
end
Out[12]:
In [25]:
@code_warntype f_barrier()
아래 f_inbuilt는 warning이 없다.
In [26]:
@code_warntype f_inner()
In [19]:
@btime f_barrier()
@btime f_inbuilt()
Out[19]:
solution 2 is to provide the types of extracted columns \ it is simpler but there are cases in which you will not know these types \ This example assumes that you have DataFrames master at least from August 31, 2018
In [16]:
function f_typed()
Random.seed!(1)
x = DataFrame(rand(1_000_000,2),:auto)
y::Vector{Float64}, z::Vector{Float64} = x[!,1], x[!,2]
p = 0.0
for i in 1:nrow(x)
p += y[i]*z[i]
end
p
end
Out[16]:
In [18]:
@btime f_typed()
Out[18]:
아래 그림에서 보면 p,y,z가 Float64 type으로 변수 타입이 확정되어 있다.\
따라서 Any type 보다는 처리가 훨씬 빠르다\
In [27]:
@code_warntype f_typed()
In general for tall and narrow tables it is often useful to use Tables.rowtable,Tables.columntable or Tables.namedtupleiterator for intermediate processing of data in a type-stable way.
Consider using delayed DataFrame creation technique¶
also notice the difference in performance between copying vs non-copying data frame creation
In [30]:
function f1()
# we work with a DataFrame directly
x = DataFrame([Vector{Float64}(undef,10^4) for i in 1:100], :auto,
copycols=false)
for c in 1:ncol(x)
d = x[!,c]
for r in 1:nrow(x)
d[r] = rand()
end
end
x
end
function f1a()
# we work with a DataFrame directly
x = DataFrame([Vector{Float64}(undef, 10^4) for i in 1:100], :auto)
for c in 1:ncol(x)
d = x[!,c]
for r in 1:nrow(x)
d[r] = rand()
end
end
end
function f2()
x = Vector{Any}(undef,100)
for c in 1:length(x)
d = Vector{Float64}(undef, 10^4)
for r in 1:length(d)
d[r] = rand()
end
x[c] = d
end
# we delay creation of DataFrame after we have our job done
DataFrame(x, :auto, copycols=false)
end
function f2a()
x = Vector{Any}(undef, 100)
for c in 1:length(x)
d = Vector{Float64}(undef, 10^4)
for r in 1:length(d)
d[r] = rand()
end
x[c] = d
end
# we delay creation of DataFrame after we have our job done
DataFrame(x, :auto)
end
Out[30]:
In [31]:
@btime f1();
@btime f1a();
@btime f2();
@btime f2a();
You can add rows to a DataFrame in place and it is fast¶
In [99]:
x = DataFrame(rand(10^6,5), :auto)
y = DataFrame(transpose(1.0:5.0), :auto)
z = [1.0:5.0;];
# creates a new DataFrame - slow
@btime vcat($x, $y)
# in place - fast
@btime append!($x,$y)
# reset to the same starting point
x = DataFrame(rand(10^6,5), :auto)
# add a single row in place - fast
@btime push!($x, $z)
Out[99]:
In [100]:
transpose(1.0:5.0)
Out[100]:
In [101]:
DataFrame(transpose(1:10), :auto)
Out[101]:
In [102]:
DataFrame([1:10...]', :auto)
Out[102]:
In [103]:
[1:10...]'
Out[103]:
In [104]:
[1:5...]'
Out[104]:
Insert row in specific index
In [105]:
insert!.(eachcol(x),1,[1:5;]);
x
Out[105]:
Allowing missing as well as categorical shows down computations¶
In [106]:
using StatsBase
Uses countmap function to test performance
In [135]:
countmap(rand([1:10;],10000))
Out[135]:
In [107]:
function test(data)
println(eltype(data))
x = rand(data, 10^6)
y = categorical(x)
println(" raw:")
@btime countmap($x)
println(" categorical:")
@btime countmap($y)
nothing
end
Out[107]:
In [108]:
test(1:10)
In [109]:
test([randstring() for i in 1:10])
In [136]:
test(allowmissing(1:10))
In [137]:
test(allowmissing([randstring() for i in 1:10]))
When aggregating use column selector and prefer integer, categorical, or pooled array grouping variable¶
In [138]:
df = DataFrame(x=rand('a':'d', 10^7), y=1)
Out[138]:
In [139]:
gdf = groupby(df,:x)
Out[139]:
Traditional syntax, slow
In [141]:
@btime combine(v->sum(v.y), $gdf)
Out[141]:
Use column selector to speed up
In [142]:
@btime combine($gdf,:y=>sum)
Out[142]:
In [143]:
transform!(df, :x=>categorical=>:x)
Out[143]:
In [144]:
gdf = groupby(df,:x)
Out[144]:
In [146]:
@btime combine($gdf, :y=>sum)
Out[146]:
In [147]:
transform!(df, :x=>PooledArray{Char}=>:x)
Out[147]:
In [148]:
gdf = groupby(df,:x)
Out[148]:
In [149]:
@btime combine($gdf, :y=>sum)
Out[149]:
Use views instead of materializing a new DataFrame¶
In [150]:
x = DataFrame(rand(100,1000), :auto)
Out[150]:
In [151]:
@btime $x[1:1,:]
Out[151]:
In [152]:
@btime $x[1,:]
Out[152]:
In [153]:
@btime view($x,1:1,:)
Out[153]:
In [154]:
@btime $x[1:1,1:20]
Out[154]:
In [155]:
@btime $x[1,1:20]
Out[155]:
In [156]:
@btime view($x,1:1,1:20)
Out[156]:
In [ ]: