\ubcc0\uacbd \uc0ac\ud56d\uc740 \uc774\uc804 MNIST updated\uc640 \ub3d9\uc77c\ud55c \ub0b4\uc6a9\uc784
accuracy\ub97c CPU\uc5d0\uc11c GPU\uc5d0\uc11c \uacc4\uc0b0\ud568<\/p>\n\n\n\n
cifar10_gpu_minibatch2.jl<\/p>\n\n\n\n
using Random\nusing BSON\nusing BSON: @save,@load\nusing Logging\nusing Dates\nusing NNlib\nusing CuArrays\nusing CUDAdrv\nusing CUDAnative: device!\nusing Flux, Metalhead, Statistics\nusing Flux: onehotbatch, onecold, crossentropy,logitcrossentropy, throttle, OneHotMatrix\nusing Metalhead: trainimgs\nusing Images: channelview\nusing Statistics: mean\nusing Base.Iterators: partition\n\nworking_path = dirname(@__FILE__)\nfile_path(file_name) = joinpath(working_path,file_name)\ninclude(file_path(\"cmd_parser.jl\"))\n\nmodel_file = file_path(\"cifar10_vgg16_model2.bson\")\n\n# Get arguments\nparsed_args = CmdParser.parse_commandline()\n\nepochs = parsed_args[\"epochs\"]\nbatch_size = parsed_args[\"batch\"]\nuse_saved_model = parsed_args[\"model\"]\ngpu_device = parsed_args[\"gpu\"]\ncreate_log_file = parsed_args[\"log\"]\n\n# epochs = 2\n# batch_size = 1000\n# use_saved_model = false\n# gpu_device = 2\n# create_log_file = false\n\n\nif create_log_file\n log_file =\".\/cifar10_vgg16_$(Dates.format(now(),\"yyyymmdd-HHMMSS\")).log\"\n log = open(log_file, \"w+\")\nelse\n log = stdout\nend\nglobal_logger(ConsoleLogger(log))\n\n@info \"Start - $(now())\";flush(log)\n\n@info \"=============== Arguments ===============\"\n@info \"epochs=$(epochs)\"\n@info \"batch_size=$(batch_size)\"\n@info \"use_saved_model=$(use_saved_model)\"\n@info \"gpu_device=$(gpu_device)\"\n@info \"=========================================\";flush(log)\n\n# Very important : this prevent loss NaN\n\u03f5 = 1.0f-32\n\n\n# use 1nd GPU : default\n#CUDAnative.device!(0)\ndevice!(gpu_device)\nCuArrays.allowscalar(false)\n\n@info \"Config VGG16, VGG19 models ...\";flush(log)\n\nacc = 0; epoch = 0\nif use_saved_model && isfile(model_file) && filesize(model_file) > 0\n # flush : \ubc84\ud37c\ub9c1 \uc5c6\uc774 \uc989\uac01 log\ub97c \ud30c\uc77c \ub610\ub294 console\uc5d0 write\ud558\ub3c4\ub85d \ud568\n @info \"Load saved model $(model_file) ...\";flush(log)\n # model : @save\uc2dc \uc0ac\uc6a9\ud55c object\uba85\n @load model_file model acc epoch\n m = model |> gpu;\n @info \" -> accuracy : $(acc), epochs : $(epoch)\";flush(log)\nelse\n @info \"Create new model ...\";flush(log)\n # VGG16 and VGG19 models\n vgg16() = Chain(\n Conv((3, 3), 3 => 64, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(64),\n Conv((3, 3), 64 => 64, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(64),\n MaxPool((2,2)),\n Conv((3, 3), 64 => 128, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(128),\n Conv((3, 3), 128 => 128, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(128),\n MaxPool((2,2)),\n Conv((3, 3), 128 => 256, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(256),\n Conv((3, 3), 256 => 256, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(256),\n Conv((3, 3), 256 => 256, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(256),\n MaxPool((2,2)),\n Conv((3, 3), 256 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n MaxPool((2,2)),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n MaxPool((2,2)),\n x -> reshape(x, :, size(x, 4)),\n Dense(512, 4096, relu),\n Dropout(0.5),\n Dense(4096, 4096, relu),\n Dropout(0.5),\n Dense(4096, 10),\n softmax)\n\n vgg19() = Chain(\n Conv((3, 3), 3 => 64, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(64),\n Conv((3, 3), 64 => 64, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(64),\n MaxPool((2,2)),\n Conv((3, 3), 64 => 128, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(128),\n Conv((3, 3), 128 => 128, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(128),\n MaxPool((2,2)),\n Conv((3, 3), 128 => 256, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(256),\n Conv((3, 3), 256 => 256, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(256),\n Conv((3, 3), 256 => 256, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(256),\n Conv((3, 3), 256 => 256, relu, pad=(1, 1), stride=(1, 1)),\n MaxPool((2,2)),\n Conv((3, 3), 256 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n MaxPool((2,2)),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n BatchNorm(512),\n Conv((3, 3), 512 => 512, relu, pad=(1, 1), stride=(1, 1)),\n MaxPool((2,2)),\n x -> reshape(x, :, size(x, 4)),\n Dense(512, 4096, relu),\n Dropout(0.5),\n Dense(4096, 4096, relu),\n Dropout(0.5),\n Dense(4096, 10),\n softmax)\n\n m = vgg16() |> gpu;\nend\ngetarray(X) = Float32.(permutedims(channelview(X), (2, 3, 1)))\n\n\n@info \"Data download and preparing ...\";flush(log)\nfunction make_minibatch(imgs,labels,batch_size)\n data_set = [(cat(imgs[i]..., dims = 4),\n labels[:,i])\n for i in partition(1:length(imgs), batch_size)]\n return data_set\nend\n\nX = trainimgs(CIFAR10)\n# Training\uc6a9 \ub370\uc774\ud130 \uc900\ube44\n# \uc774\ubbf8\uc9c0 \ucc44\ub110 \ubd84\ub9ac \ubc0f \uc7ac\ubc30\uc5f4, training\uc6a9\uc73c\ub85c 60,000\uac1c\uc911 50,000\uac1c\ub97c \uc0ac\uc6a9\ud55c\ub2e4.\ntrain_idxs = 1:49000\ntrain_imgs = [getarray(X[i].img) for i in train_idxs]\ntrain_labels = onehotbatch([X[i].ground_truth.class for i in train_idxs],1:10)\ntrain_set = make_minibatch(train_imgs,train_labels,batch_size)\n\nverify_idxs = 49001:50000\nverify_imgs = cat([getarray(X[i].img) for i in verify_idxs]..., dims = 4)\nverify_labels = onehotbatch([X[i].ground_truth.class for i in verify_idxs],1:10)\nverify_set = [(verify_imgs,verify_labels)]\n\n# Fetch the test data from Metalhead and get it into proper shape.\n# CIFAR-10 does not specify a verify set so valimgs fetch the testdata instead of testimgs\ntX = valimgs(CIFAR10)\ntest_idxs = 1:10000\ntest_imgs = [getarray(tX[i].img) for i in test_idxs]\ntest_labels = onehotbatch([tX[i].ground_truth.class for i in test_idxs], 1:10)\ntest_set = make_minibatch(test_imgs,test_labels,batch_size)\n# Defining the loss and accuracy functions\n\n@info \"VGG16 models instantiation ...\";flush(log)\n\n\nloss(x, y) = crossentropy(m(x) .+ \u03f5, y)\n\ncompare(y::OneHotMatrix, y\u0302) = maximum(y\u0302,dims = 1) .== maximum(y .* y\u0302, dims = 1)\nfunction accuracy(data_set)\n batch_size = size(data_set[1][1])[end]\n l = length(data_set)*batch_size\n s = 0f0\n for (x,y::OneHotMatrix) in data_set\n s += sum(compare(y|>gpu,m(x|>gpu)))\n end\n return s\/l\nend\n\n\n# loss(x, y) = crossentropy(m(x) .+ \u03f5, y)\n# function accuracy(data_set)\n# batch_size = size(data_set[1][1])[end]\n# l = length(data_set)*batch_size\n# s = 0f0\n# for (x,y) in data_set\n# s += sum((onecold(m(x|>gpu) |> cpu) .== onecold(y|>cpu)))\n# end\n# return s\/l\n# end\n\n# Make sure our is nicely precompiled befor starting our training loop\n# train_set[1][1] : (28,28,1,batch_size)\n@info \"Model pre-compile...\";flush(log)\nm(train_set[1][1] |> gpu)\n\n# Defining the callback and the optimizer\n# evalcb = throttle(() -> @info(accuracy(verify_set)), 10)\nopt = ADAM(0.001)\n\n@info \"Training model...\";flush(log)\nbest_acc = acc\nlast_improvement = epoch\n# used for plots\nfor epoch_idx in 1+epoch:(epochs+=epoch)\n accs = Array{Float32}(undef,0)\n global best_acc, last_improvement\n train_set_len = length(train_set)\n shuffle_idxs = collect(1:train_set_len)\n shuffle!(shuffle_idxs)\n\n for (idx,data_idx) in enumerate(shuffle_idxs)\n (x,y) = train_set[data_idx]\n # We augment `x` a little bit here, adding in random noise\n x = (x .+ \u03f5*randn(eltype(x),size(x))) |> gpu;\n y = y|> gpu;\n Flux.train!(loss,params(m),[(x,y)],opt)\n #Flux.train!(loss,params(m),[(x,y)],opt,cb = evalcb)\n v_acc = accuracy(verify_set)\n @info \"Epoch# $(epoch_idx)\/$(epochs) - #$(idx)\/$(train_set_len) loss: $(loss(x,y)), accuracy: $(v_acc)\";flush(log)\n # @info \"Epoch# $(epoch_idx)\/$(epochs) - #$(idx)\/$(train_set_len) accuracy: $(v_acc)\";flush(log)\n push!(accs,v_acc)\n end # for\n\n m_acc = mean(accs)\n @info \" -> Verify accuracy(mean) : $(m_acc)\";flush(log)\n test_acc = accuracy(test_set)\n @info \"Test accuracy : $(test_acc)\";flush(log)\n\n # If our accuracy is good enough, quit out.\n if test_acc >= 0.98\n @info \" -> Early-exiting: We reached our target accuracy of 98%\";flush(log)\n model = m |> cpu;acc = test_acc;epoch = epoch_idx\n @save model_file model acc epoch\n break\n end\n\n # If this is the best accuracy we've seen so far, save the model out\n if test_acc >= best_acc\n @info \" -> New best accuracy! saving model out to $(model_file)\"; flush(log)\n model = m |> cpu;acc = test_acc;epoch = epoch_idx\n # @save,@load \uc2dc \uac19\uc740 \uc774\ub984\uc744 \uc0ac\uc6a9\ud574\uc57c \ud568, \uc5ec\uae30\uc11c\ub294 \"model\"\uc744 \uc0ac\uc6a9\ud568\n @save model_file model acc epoch\n best_acc = test_acc\n last_improvement = epoch_idx\n end\n\n # If we haven't seen improvement in 5 epochs, drop out learning rate:\n if epoch_idx - last_improvement >= 5 && opt.eta > 1e-6\n opt.eta \/= 10.0\n @info \" -> Haven't improved in a while, dropping learning rate to $(opt.eta)!\";flush(log)\n # After dropping learning rate, give it a few epochs to improve\n last_improvement = epoch_idx\n end\n\n if epoch_idx - last_improvement >= 10\n @info \" -> We're calling this converged.\"; flush(log)\n break\n end\nend # end of for\n\n@info \"End - $(now())\"\nif create_log_file\n close(log)\nend<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"\ubcc0\uacbd \uc0ac\ud56d\uc740 \uc774\uc804 MNIST updated\uc640 \ub3d9\uc77c\ud55c \ub0b4\uc6a9\uc784accuracy\ub97c CPU\uc5d0\uc11c GPU\uc5d0\uc11c \uacc4\uc0b0\ud568 cifar10_gpu_minibatch2.jl<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"site-sidebar-layout":"default","site-content-layout":"default","ast-site-content-layout":"","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"default","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"default","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":""},"categories":[18,21],"tags":[],"_links":{"self":[{"href":"https:\/\/julialang.kr\/index.php?rest_route=\/wp\/v2\/posts\/2561"}],"collection":[{"href":"https:\/\/julialang.kr\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/julialang.kr\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/julialang.kr\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/julialang.kr\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2561"}],"version-history":[{"count":1,"href":"https:\/\/julialang.kr\/index.php?rest_route=\/wp\/v2\/posts\/2561\/revisions"}],"predecessor-version":[{"id":2562,"href":"https:\/\/julialang.kr\/index.php?rest_route=\/wp\/v2\/posts\/2561\/revisions\/2562"}],"wp:attachment":[{"href":"https:\/\/julialang.kr\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2561"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/julialang.kr\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2561"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/julialang.kr\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2561"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}