YOLO(Game)

 

 

In [1]:

from google.colab import drive
drive.mount('/content/drive')

 

Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).

In [2]:

!ls -la "/content/drive/MyDrive/darknet"

 

total 32
drwx------ 2 root root 4096 Dec 23 03:16 backup
drwx------ 2 root root 4096 Dec 23 02:33 bin
drwx------ 2 root root 4096 Dec 23 03:04 cfg
drwx------ 2 root root 4096 Dec 23 02:16 cuDNN
drwx------ 2 root root 4096 Dec 23 03:04 custom
drwx------ 2 root root 4096 Dec 23 03:04 data
drwx------ 2 root root 4096 Dec 23 02:16 .ipynb_checkpoints
drwx------ 2 root root 4096 Dec 23 02:36 weights

In [3]:

!tar -xzvf /content/drive/MyDrive/darknet/cuDNN/cudnn-10.1-linux-x64-v7.6.5.32.tgz -C /usr/local/
!chmod a+r /usr/local/cuda/include/cudnn.h

!cat /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2

 

cuda/include/cudnn.h
cuda/NVIDIA_SLA_cuDNN_Support.txt
cuda/lib64/libcudnn.so
cuda/lib64/libcudnn.so.7
cuda/lib64/libcudnn.so.7.6.5
cuda/lib64/libcudnn_static.a
#define CUDNN_MAJOR 7
#define CUDNN_MINOR 6
#define CUDNN_PATCHLEVEL 5
--
#define CUDNN_VERSION (CUDNN_MAJOR * 1000 + CUDNN_MINOR * 100 + CUDNN_PATCHLEVEL)

#include "driver_types.h"

In [4]:

!ls -la /usr/local/cuda/include/cudnn.h

 

-r--r--r-- 1 root root 134857 Sep 26  2019 /usr/local/cuda/include/cudnn.h

In [5]:

import os
if not os.path.exists('darknet'):
    os.makedirs('darknet')
%cd darknet
%ls

 

/content/darknet

In [6]:

!ls -la '/content/drive/MyDrive/darknet/bin/darknet'

!cp /content/drive/MyDrive/darknet/bin/darknet ./darknet

!chmod +x ./darknet

 

-rw------- 1 root root 3213624 Dec 23 02:35 /content/drive/MyDrive/darknet/bin/darknet

In [7]:

!cp -r '/content/drive/MyDrive/darknet/weights' .
!cp -r '/content/drive/MyDrive/darknet/cfg' .
!cp -ar '/content/drive/MyDrive/darknet/data' .
%ls

 

cfg/  darknet*  data/  weights/

In [8]:

def imShow(path):
  import cv2
  import matplotlib.pyplot as plt
  %matplotlib inline

  image = cv2.imread(path)
  height, width = image.shape[:2]
  resized_image = cv2.resize(image,(3*width, 3*height), interpolation = cv2.INTER_CUBIC)

  fig = plt.gcf()
  fig.set_size_inches(18, 10)
  plt.axis("off")
  #plt.rcParams['figure.figsize'] = [10, 5]
  plt.imshow(cv2.cvtColor(resized_image, cv2.COLOR_BGR2RGB))
  plt.show()
  
def upload():
  from google.colab import files
  uploaded = files.upload() 
  for name, data in uploaded.items():
    with open(name, 'wb') as f:
      f.write(data)
      print ('saved file', name)
      
def download(path):
  from google.colab import files
  files.download(path)

In [9]:

!cp -r "/content/drive/MyDrive/darknet/custom/game_yolo" .

In [ ]:

!./darknet detector train game_yolo/custom_data.data game_yolo/custom-train-yolo.cfg weights/darknet53.conv.74 -dont_show

In [ ]:

if os.path.exists('./backup'):
    !cp -r ./backup "/content/drive/MyDrive/darknet"  

if not os.path.exists('./backup'):
    os.makedirs('./backup')
    !cp -r "/content/drive/MyDrive/darknet/backup/custom-train-yolo_backup.weights" ./backup

In [12]:

!./darknet detector train game_yolo/custom_data.data game_yolo/custom-train-yolo.cfg backup/custom-train-yolo_backup.weights -dont_show

 

스트리밍 출력 내용이 길어서 마지막 5000줄이 삭제되었습니다.
Loaded: 0.000056 seconds

 10535: 0.011961, 0.014427 avg loss, 0.000100 rate, 5.120417 seconds, 337120 images
Loaded: 0.000043 seconds

 10536: 0.014765, 0.014460 avg loss, 0.000100 rate, 5.119435 seconds, 337152 images
Loaded: 0.000047 seconds

 10537: 0.020896, 0.015104 avg loss, 0.000100 rate, 5.132697 seconds, 337184 images
Loaded: 0.000066 seconds

 10538: 0.020479, 0.015642 avg loss, 0.000100 rate, 5.131049 seconds, 337216 images
Loaded: 0.000041 seconds

 10539: 0.020679, 0.016145 avg loss, 0.000100 rate, 5.129308 seconds, 337248 images
Loaded: 0.000040 seconds

 10540: 0.019425, 0.016473 avg loss, 0.000100 rate, 5.124556 seconds, 337280 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.000044 seconds

 10541: 0.009277, 0.015754 avg loss, 0.000100 rate, 2.707848 seconds, 337312 images
Loaded: 0.000045 seconds

 10542: 0.010988, 0.015277 avg loss, 0.000100 rate, 2.801127 seconds, 337344 images
Loaded: 0.000067 seconds

 10543: 0.009980, 0.014747 avg loss, 0.000100 rate, 2.800858 seconds, 337376 images
Loaded: 0.000042 seconds

 10544: 0.011010, 0.014374 avg loss, 0.000100 rate, 2.802643 seconds, 337408 images
Loaded: 0.000052 seconds

 10545: 0.014210, 0.014357 avg loss, 0.000100 rate, 2.803413 seconds, 337440 images
Loaded: 0.000041 seconds

 10546: 0.012944, 0.014216 avg loss, 0.000100 rate, 2.803726 seconds, 337472 images
Loaded: 0.000037 seconds

 10547: 0.021761, 0.014970 avg loss, 0.000100 rate, 2.808892 seconds, 337504 images
Loaded: 0.000046 seconds

 10548: 0.009895, 0.014463 avg loss, 0.000100 rate, 2.807305 seconds, 337536 images
Loaded: 0.000042 seconds

 10549: 0.010311, 0.014048 avg loss, 0.000100 rate, 2.809874 seconds, 337568 images
Loaded: 0.000040 seconds

 10550: 0.012754, 0.013918 avg loss, 0.000100 rate, 2.807887 seconds, 337600 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.046074 seconds

 10551: 0.026938, 0.015220 avg loss, 0.000100 rate, 1.853901 seconds, 337632 images
Loaded: 0.000066 seconds

 10552: 0.014593, 0.015158 avg loss, 0.000100 rate, 1.937303 seconds, 337664 images
Loaded: 0.000068 seconds

 10553: 0.019120, 0.015554 avg loss, 0.000100 rate, 1.927001 seconds, 337696 images
Loaded: 0.000044 seconds

 10554: 0.027432, 0.016742 avg loss, 0.000100 rate, 1.927090 seconds, 337728 images
Loaded: 0.000038 seconds

 10555: 0.017476, 0.016815 avg loss, 0.000100 rate, 1.939499 seconds, 337760 images
Loaded: 0.000036 seconds

 10556: 0.018538, 0.016987 avg loss, 0.000100 rate, 1.929336 seconds, 337792 images
Loaded: 0.000042 seconds

 10557: 0.015584, 0.016847 avg loss, 0.000100 rate, 1.931714 seconds, 337824 images
Loaded: 0.000043 seconds

 10558: 0.025378, 0.017700 avg loss, 0.000100 rate, 1.922949 seconds, 337856 images
Loaded: 0.000058 seconds

 10559: 0.019833, 0.017913 avg loss, 0.000100 rate, 1.920832 seconds, 337888 images
Loaded: 0.000043 seconds

 10560: 0.013559, 0.017478 avg loss, 0.000100 rate, 1.922978 seconds, 337920 images
Resizing
608 x 608 
 try to allocate additional workspace_size = 108.69 MB 
 CUDA allocate done! 
Loaded: 0.000047 seconds

 10561: 0.017592, 0.017489 avg loss, 0.000100 rate, 4.925764 seconds, 337952 images
Loaded: 0.000041 seconds

 10562: 0.027785, 0.018519 avg loss, 0.000100 rate, 5.095933 seconds, 337984 images
Loaded: 0.000045 seconds

 10563: 0.011251, 0.017792 avg loss, 0.000100 rate, 5.108922 seconds, 338016 images
Loaded: 0.000046 seconds

 10564: 0.020892, 0.018102 avg loss, 0.000100 rate, 5.115345 seconds, 338048 images
Loaded: 0.000078 seconds

 10565: 0.011566, 0.017449 avg loss, 0.000100 rate, 5.113529 seconds, 338080 images
Loaded: 0.000042 seconds

 10566: 0.021113, 0.017815 avg loss, 0.000100 rate, 5.113166 seconds, 338112 images
Loaded: 0.000070 seconds

 10567: 0.025633, 0.018597 avg loss, 0.000100 rate, 5.119201 seconds, 338144 images
Loaded: 0.000048 seconds

 10568: 0.013301, 0.018067 avg loss, 0.000100 rate, 5.117432 seconds, 338176 images
Loaded: 0.000054 seconds

 10569: 0.025559, 0.018816 avg loss, 0.000100 rate, 5.101384 seconds, 338208 images
Loaded: 0.000040 seconds

 10570: 0.009721, 0.017907 avg loss, 0.000100 rate, 5.115718 seconds, 338240 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000037 seconds

 10571: 0.008562, 0.016972 avg loss, 0.000100 rate, 3.257555 seconds, 338272 images
Loaded: 0.000087 seconds

 10572: 0.009132, 0.016188 avg loss, 0.000100 rate, 3.380252 seconds, 338304 images
Loaded: 0.000056 seconds

 10573: 0.018791, 0.016449 avg loss, 0.000100 rate, 3.380895 seconds, 338336 images
Loaded: 0.000072 seconds

 10574: 0.010331, 0.015837 avg loss, 0.000100 rate, 3.380682 seconds, 338368 images
Loaded: 0.000040 seconds

 10575: 0.014114, 0.015664 avg loss, 0.000100 rate, 3.385033 seconds, 338400 images
Loaded: 0.000045 seconds

 10576: 0.012719, 0.015370 avg loss, 0.000100 rate, 3.385866 seconds, 338432 images
Loaded: 0.000075 seconds

 10577: 0.011517, 0.014985 avg loss, 0.000100 rate, 3.386016 seconds, 338464 images
Loaded: 0.000046 seconds

 10578: 0.011524, 0.014639 avg loss, 0.000100 rate, 3.387218 seconds, 338496 images
Loaded: 0.000068 seconds

 10579: 0.009987, 0.014173 avg loss, 0.000100 rate, 3.386358 seconds, 338528 images
Loaded: 0.000046 seconds

 10580: 0.013257, 0.014082 avg loss, 0.000100 rate, 3.386940 seconds, 338560 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 10581: 0.011150, 0.013789 avg loss, 0.000100 rate, 4.169987 seconds, 338592 images
Loaded: 0.000076 seconds

 10582: 0.015273, 0.013937 avg loss, 0.000100 rate, 4.321862 seconds, 338624 images
Loaded: 0.000055 seconds

 10583: 0.010016, 0.013545 avg loss, 0.000100 rate, 4.330499 seconds, 338656 images
Loaded: 0.000072 seconds

 10584: 0.011260, 0.013316 avg loss, 0.000100 rate, 4.329586 seconds, 338688 images
Loaded: 0.000073 seconds

 10585: 0.012379, 0.013223 avg loss, 0.000100 rate, 4.327239 seconds, 338720 images
Loaded: 0.000072 seconds

 10586: 0.024287, 0.014329 avg loss, 0.000100 rate, 4.333074 seconds, 338752 images
Loaded: 0.000042 seconds

 10587: 0.015530, 0.014449 avg loss, 0.000100 rate, 4.335439 seconds, 338784 images
Loaded: 0.000042 seconds

 10588: 0.017301, 0.014734 avg loss, 0.000100 rate, 4.322890 seconds, 338816 images
Loaded: 0.000042 seconds

 10589: 0.028095, 0.016070 avg loss, 0.000100 rate, 4.334781 seconds, 338848 images
Loaded: 0.000046 seconds

 10590: 0.012608, 0.015724 avg loss, 0.000100 rate, 4.328499 seconds, 338880 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 10591: 0.016781, 0.015830 avg loss, 0.000100 rate, 1.842739 seconds, 338912 images
Loaded: 0.000076 seconds

 10592: 0.018785, 0.016125 avg loss, 0.000100 rate, 1.920526 seconds, 338944 images
Loaded: 0.000038 seconds

 10593: 0.012441, 0.015757 avg loss, 0.000100 rate, 1.919094 seconds, 338976 images
Loaded: 0.000043 seconds

 10594: 0.014040, 0.015585 avg loss, 0.000100 rate, 1.930218 seconds, 339008 images
Loaded: 0.000041 seconds

 10595: 0.014172, 0.015444 avg loss, 0.000100 rate, 1.924272 seconds, 339040 images
Loaded: 0.000075 seconds

 10596: 0.018772, 0.015777 avg loss, 0.000100 rate, 1.919663 seconds, 339072 images
Loaded: 0.000041 seconds

 10597: 0.021380, 0.016337 avg loss, 0.000100 rate, 1.921136 seconds, 339104 images
Loaded: 0.000040 seconds

 10598: 0.010931, 0.015796 avg loss, 0.000100 rate, 1.927105 seconds, 339136 images
Loaded: 0.000043 seconds

 10599: 0.016235, 0.015840 avg loss, 0.000100 rate, 1.917654 seconds, 339168 images
Loaded: 0.000049 seconds

 10600: 0.014028, 0.015659 avg loss, 0.000100 rate, 1.923753 seconds, 339200 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000071 seconds

 10601: 0.011832, 0.015276 avg loss, 0.000100 rate, 2.098455 seconds, 339232 images
Loaded: 0.000041 seconds

 10602: 0.014008, 0.015150 avg loss, 0.000100 rate, 2.166325 seconds, 339264 images
Loaded: 0.000064 seconds

 10603: 0.013242, 0.014959 avg loss, 0.000100 rate, 2.166451 seconds, 339296 images
Loaded: 0.000068 seconds

 10604: 0.014628, 0.014926 avg loss, 0.000100 rate, 2.164692 seconds, 339328 images
Loaded: 0.000042 seconds

 10605: 0.012678, 0.014701 avg loss, 0.000100 rate, 2.174013 seconds, 339360 images
Loaded: 0.000040 seconds

 10606: 0.013018, 0.014533 avg loss, 0.000100 rate, 2.176740 seconds, 339392 images
Loaded: 0.000057 seconds

 10607: 0.009336, 0.014013 avg loss, 0.000100 rate, 2.164973 seconds, 339424 images
Loaded: 0.000069 seconds

 10608: 0.010405, 0.013652 avg loss, 0.000100 rate, 2.167618 seconds, 339456 images
Loaded: 0.000046 seconds

 10609: 0.011181, 0.013405 avg loss, 0.000100 rate, 2.169249 seconds, 339488 images
Loaded: 0.000045 seconds

 10610: 0.015515, 0.013616 avg loss, 0.000100 rate, 2.165182 seconds, 339520 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000050 seconds

 10611: 0.010498, 0.013304 avg loss, 0.000100 rate, 2.456031 seconds, 339552 images
Loaded: 0.000047 seconds

 10612: 0.021528, 0.014127 avg loss, 0.000100 rate, 2.561032 seconds, 339584 images
Loaded: 0.000065 seconds

 10613: 0.014944, 0.014208 avg loss, 0.000100 rate, 2.562657 seconds, 339616 images
Loaded: 0.000049 seconds

 10614: 0.008404, 0.013628 avg loss, 0.000100 rate, 2.559640 seconds, 339648 images
Loaded: 0.000042 seconds

 10615: 0.009708, 0.013236 avg loss, 0.000100 rate, 2.563219 seconds, 339680 images
Loaded: 0.000057 seconds

 10616: 0.011078, 0.013020 avg loss, 0.000100 rate, 2.562835 seconds, 339712 images
Loaded: 0.000050 seconds

 10617: 0.019240, 0.013642 avg loss, 0.000100 rate, 2.560816 seconds, 339744 images
Loaded: 0.000060 seconds

 10618: 0.011147, 0.013393 avg loss, 0.000100 rate, 2.566614 seconds, 339776 images
Loaded: 0.000042 seconds

 10619: 0.014073, 0.013461 avg loss, 0.000100 rate, 2.566196 seconds, 339808 images
Loaded: 0.000067 seconds

 10620: 0.015125, 0.013627 avg loss, 0.000100 rate, 2.564994 seconds, 339840 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000035 seconds

 10621: 0.009824, 0.013247 avg loss, 0.000100 rate, 3.020416 seconds, 339872 images
Loaded: 0.000070 seconds

 10622: 0.007814, 0.012704 avg loss, 0.000100 rate, 3.137111 seconds, 339904 images
Loaded: 0.000072 seconds

 10623: 0.008608, 0.012294 avg loss, 0.000100 rate, 3.138912 seconds, 339936 images
Loaded: 0.000082 seconds

 10624: 0.012379, 0.012303 avg loss, 0.000100 rate, 3.143805 seconds, 339968 images
Loaded: 0.000069 seconds

 10625: 0.009752, 0.012048 avg loss, 0.000100 rate, 3.147055 seconds, 340000 images
Loaded: 0.000064 seconds

 10626: 0.011941, 0.012037 avg loss, 0.000100 rate, 3.158524 seconds, 340032 images
Loaded: 0.000072 seconds

 10627: 0.013781, 0.012211 avg loss, 0.000100 rate, 3.147544 seconds, 340064 images
Loaded: 0.000080 seconds

 10628: 0.009554, 0.011946 avg loss, 0.000100 rate, 3.143121 seconds, 340096 images
Loaded: 0.000033 seconds

 10629: 0.008530, 0.011604 avg loss, 0.000100 rate, 3.157463 seconds, 340128 images
Loaded: 0.000041 seconds

 10630: 0.009326, 0.011376 avg loss, 0.000100 rate, 3.144512 seconds, 340160 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.045847 seconds

 10631: 0.009399, 0.011178 avg loss, 0.000100 rate, 2.467446 seconds, 340192 images
Loaded: 0.000043 seconds

 10632: 0.009166, 0.010977 avg loss, 0.000100 rate, 2.572335 seconds, 340224 images
Loaded: 0.000066 seconds

 10633: 0.013030, 0.011182 avg loss, 0.000100 rate, 2.569254 seconds, 340256 images
Loaded: 0.000035 seconds

 10634: 0.017314, 0.011796 avg loss, 0.000100 rate, 2.572079 seconds, 340288 images
Loaded: 0.000053 seconds

 10635: 0.012443, 0.011860 avg loss, 0.000100 rate, 2.573494 seconds, 340320 images
Loaded: 0.000046 seconds

 10636: 0.013674, 0.012042 avg loss, 0.000100 rate, 2.570014 seconds, 340352 images
Loaded: 0.000067 seconds

 10637: 0.013715, 0.012209 avg loss, 0.000100 rate, 2.572079 seconds, 340384 images
Loaded: 0.000043 seconds

 10638: 0.024663, 0.013454 avg loss, 0.000100 rate, 2.559344 seconds, 340416 images
Loaded: 0.000041 seconds

 10639: 0.016622, 0.013771 avg loss, 0.000100 rate, 2.568599 seconds, 340448 images
Loaded: 0.000044 seconds

 10640: 0.010462, 0.013440 avg loss, 0.000100 rate, 2.561850 seconds, 340480 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.045317 seconds

 10641: 0.016220, 0.013718 avg loss, 0.000100 rate, 1.850161 seconds, 340512 images
Loaded: 0.000037 seconds

 10642: 0.034461, 0.015793 avg loss, 0.000100 rate, 1.933552 seconds, 340544 images
Loaded: 0.000041 seconds

 10643: 0.014306, 0.015644 avg loss, 0.000100 rate, 1.924793 seconds, 340576 images
Loaded: 0.000072 seconds

 10644: 0.015914, 0.015671 avg loss, 0.000100 rate, 1.930347 seconds, 340608 images
Loaded: 0.000049 seconds

 10645: 0.014314, 0.015535 avg loss, 0.000100 rate, 1.925390 seconds, 340640 images
Loaded: 0.000045 seconds

 10646: 0.015169, 0.015499 avg loss, 0.000100 rate, 1.925565 seconds, 340672 images
Loaded: 0.000071 seconds

 10647: 0.014503, 0.015399 avg loss, 0.000100 rate, 1.925413 seconds, 340704 images
Loaded: 0.000043 seconds

 10648: 0.017590, 0.015618 avg loss, 0.000100 rate, 1.928243 seconds, 340736 images
Loaded: 0.000061 seconds

 10649: 0.014794, 0.015536 avg loss, 0.000100 rate, 1.932244 seconds, 340768 images
Loaded: 0.000045 seconds

 10650: 0.015719, 0.015554 avg loss, 0.000100 rate, 1.925656 seconds, 340800 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 10651: 0.022463, 0.016245 avg loss, 0.000100 rate, 2.097266 seconds, 340832 images
Loaded: 0.000043 seconds

 10652: 0.013815, 0.016002 avg loss, 0.000100 rate, 2.173144 seconds, 340864 images
Loaded: 0.000055 seconds

 10653: 0.013702, 0.015772 avg loss, 0.000100 rate, 2.169221 seconds, 340896 images
Loaded: 0.000045 seconds

 10654: 0.024724, 0.016667 avg loss, 0.000100 rate, 2.165194 seconds, 340928 images
Loaded: 0.000068 seconds

 10655: 0.012989, 0.016299 avg loss, 0.000100 rate, 2.174251 seconds, 340960 images
Loaded: 0.000070 seconds

 10656: 0.012025, 0.015872 avg loss, 0.000100 rate, 2.173667 seconds, 340992 images
Loaded: 0.000072 seconds

 10657: 0.017002, 0.015985 avg loss, 0.000100 rate, 2.174517 seconds, 341024 images
Loaded: 0.000069 seconds

 10658: 0.011687, 0.015555 avg loss, 0.000100 rate, 2.179832 seconds, 341056 images
Loaded: 0.000079 seconds

 10659: 0.019924, 0.015992 avg loss, 0.000100 rate, 2.169466 seconds, 341088 images
Loaded: 0.000039 seconds

 10660: 0.027430, 0.017136 avg loss, 0.000100 rate, 2.169975 seconds, 341120 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 10661: 0.014748, 0.016897 avg loss, 0.000100 rate, 3.267384 seconds, 341152 images
Loaded: 0.000043 seconds

 10662: 0.010331, 0.016240 avg loss, 0.000100 rate, 3.394263 seconds, 341184 images
Loaded: 0.000046 seconds

 10663: 0.007323, 0.015349 avg loss, 0.000100 rate, 3.382804 seconds, 341216 images
Loaded: 0.000041 seconds

 10664: 0.013126, 0.015126 avg loss, 0.000100 rate, 3.379734 seconds, 341248 images
Loaded: 0.000085 seconds

 10665: 0.008768, 0.014491 avg loss, 0.000100 rate, 3.388616 seconds, 341280 images
Loaded: 0.000069 seconds

 10666: 0.010885, 0.014130 avg loss, 0.000100 rate, 3.387372 seconds, 341312 images
Loaded: 0.000056 seconds

 10667: 0.008039, 0.013521 avg loss, 0.000100 rate, 3.383854 seconds, 341344 images
Loaded: 0.000041 seconds

 10668: 0.011917, 0.013361 avg loss, 0.000100 rate, 3.392385 seconds, 341376 images
Loaded: 0.000078 seconds

 10669: 0.022667, 0.014291 avg loss, 0.000100 rate, 3.389961 seconds, 341408 images
Loaded: 0.000071 seconds

 10670: 0.012479, 0.014110 avg loss, 0.000100 rate, 3.390031 seconds, 341440 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.054119 seconds

 10671: 0.010435, 0.013743 avg loss, 0.000100 rate, 3.268813 seconds, 341472 images
Loaded: 0.000049 seconds

 10672: 0.008286, 0.013197 avg loss, 0.000100 rate, 3.388003 seconds, 341504 images
Loaded: 0.000074 seconds

 10673: 0.023001, 0.014177 avg loss, 0.000100 rate, 3.388020 seconds, 341536 images
Loaded: 0.000094 seconds

 10674: 0.012899, 0.014049 avg loss, 0.000100 rate, 3.388778 seconds, 341568 images
Loaded: 0.000043 seconds

 10675: 0.014005, 0.014045 avg loss, 0.000100 rate, 3.384473 seconds, 341600 images
Loaded: 0.000081 seconds

 10676: 0.009642, 0.013605 avg loss, 0.000100 rate, 3.394831 seconds, 341632 images
Loaded: 0.000079 seconds

 10677: 0.013726, 0.013617 avg loss, 0.000100 rate, 3.392666 seconds, 341664 images
Loaded: 0.000044 seconds

 10678: 0.014234, 0.013679 avg loss, 0.000100 rate, 3.389963 seconds, 341696 images
Loaded: 0.000042 seconds

 10679: 0.011856, 0.013496 avg loss, 0.000100 rate, 3.387343 seconds, 341728 images
Loaded: 0.000066 seconds

 10680: 0.009860, 0.013133 avg loss, 0.000100 rate, 3.397461 seconds, 341760 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.122464 seconds

 10681: 0.011753, 0.012995 avg loss, 0.000100 rate, 2.471778 seconds, 341792 images
Loaded: 0.000074 seconds

 10682: 0.056893, 0.017385 avg loss, 0.000100 rate, 2.559923 seconds, 341824 images
Loaded: 0.000040 seconds

 10683: 0.015659, 0.017212 avg loss, 0.000100 rate, 2.561087 seconds, 341856 images
Loaded: 0.000050 seconds

 10684: 0.014851, 0.016976 avg loss, 0.000100 rate, 2.566739 seconds, 341888 images
Loaded: 0.000055 seconds

 10685: 0.009468, 0.016225 avg loss, 0.000100 rate, 2.562906 seconds, 341920 images
Loaded: 0.000071 seconds

 10686: 0.015638, 0.016166 avg loss, 0.000100 rate, 2.566110 seconds, 341952 images
Loaded: 0.000054 seconds

 10687: 0.013607, 0.015910 avg loss, 0.000100 rate, 2.565083 seconds, 341984 images
Loaded: 0.000065 seconds

 10688: 0.011964, 0.015516 avg loss, 0.000100 rate, 2.565285 seconds, 342016 images
Loaded: 0.000043 seconds

 10689: 0.009734, 0.014938 avg loss, 0.000100 rate, 2.563072 seconds, 342048 images
Loaded: 0.000044 seconds

 10690: 0.009406, 0.014385 avg loss, 0.000100 rate, 2.569533 seconds, 342080 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.000038 seconds

 10691: 0.013293, 0.014275 avg loss, 0.000100 rate, 2.706490 seconds, 342112 images
Loaded: 0.000069 seconds

 10692: 0.011061, 0.013954 avg loss, 0.000100 rate, 2.804357 seconds, 342144 images
Loaded: 0.000035 seconds

 10693: 0.009769, 0.013535 avg loss, 0.000100 rate, 2.811599 seconds, 342176 images
Loaded: 0.000045 seconds

 10694: 0.022536, 0.014435 avg loss, 0.000100 rate, 2.815187 seconds, 342208 images
Loaded: 0.000070 seconds

 10695: 0.011998, 0.014192 avg loss, 0.000100 rate, 2.806015 seconds, 342240 images
Loaded: 0.000050 seconds

 10696: 0.014487, 0.014221 avg loss, 0.000100 rate, 2.811482 seconds, 342272 images
Loaded: 0.000042 seconds

 10697: 0.007349, 0.013534 avg loss, 0.000100 rate, 2.811540 seconds, 342304 images
Loaded: 0.000044 seconds

 10698: 0.009496, 0.013130 avg loss, 0.000100 rate, 2.804177 seconds, 342336 images
Loaded: 0.000048 seconds

 10699: 0.011581, 0.012975 avg loss, 0.000100 rate, 2.802746 seconds, 342368 images
Loaded: 0.000042 seconds

 10700: 0.010442, 0.012722 avg loss, 0.000100 rate, 2.803624 seconds, 342400 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 10701: 0.014741, 0.012924 avg loss, 0.000100 rate, 2.084273 seconds, 342432 images
Loaded: 0.000067 seconds

 10702: 0.016297, 0.013261 avg loss, 0.000100 rate, 2.163123 seconds, 342464 images
Loaded: 0.000070 seconds

 10703: 0.012611, 0.013196 avg loss, 0.000100 rate, 2.167862 seconds, 342496 images
Loaded: 0.000041 seconds

 10704: 0.020785, 0.013955 avg loss, 0.000100 rate, 2.166739 seconds, 342528 images
Loaded: 0.000042 seconds

 10705: 0.012089, 0.013768 avg loss, 0.000100 rate, 2.170821 seconds, 342560 images
Loaded: 0.000043 seconds

 10706: 0.013006, 0.013692 avg loss, 0.000100 rate, 2.170683 seconds, 342592 images
Loaded: 0.000066 seconds

 10707: 0.011848, 0.013508 avg loss, 0.000100 rate, 2.166205 seconds, 342624 images
Loaded: 0.000067 seconds

 10708: 0.010749, 0.013232 avg loss, 0.000100 rate, 2.170154 seconds, 342656 images
Loaded: 0.000068 seconds

 10709: 0.008447, 0.012753 avg loss, 0.000100 rate, 2.155124 seconds, 342688 images
Loaded: 0.000067 seconds

 10710: 0.016159, 0.013094 avg loss, 0.000100 rate, 2.161314 seconds, 342720 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000061 seconds

 10711: 0.016405, 0.013425 avg loss, 0.000100 rate, 3.261603 seconds, 342752 images
Loaded: 0.000045 seconds

 10712: 0.010798, 0.013162 avg loss, 0.000100 rate, 3.378829 seconds, 342784 images
Loaded: 0.000077 seconds

 10713: 0.012962, 0.013142 avg loss, 0.000100 rate, 3.366072 seconds, 342816 images
Loaded: 0.000046 seconds

 10714: 0.010250, 0.012853 avg loss, 0.000100 rate, 3.380788 seconds, 342848 images
Loaded: 0.000060 seconds

 10715: 0.011976, 0.012765 avg loss, 0.000100 rate, 3.380108 seconds, 342880 images
Loaded: 0.000045 seconds

 10716: 0.016968, 0.013186 avg loss, 0.000100 rate, 3.389442 seconds, 342912 images
Loaded: 0.000073 seconds

 10717: 0.013465, 0.013214 avg loss, 0.000100 rate, 3.389836 seconds, 342944 images
Loaded: 0.000071 seconds

 10718: 0.014419, 0.013334 avg loss, 0.000100 rate, 3.394354 seconds, 342976 images
Loaded: 0.000039 seconds

 10719: 0.010104, 0.013011 avg loss, 0.000100 rate, 3.391371 seconds, 343008 images
Loaded: 0.000042 seconds

 10720: 0.011775, 0.012888 avg loss, 0.000100 rate, 3.382234 seconds, 343040 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.014159 seconds

 10721: 0.011509, 0.012750 avg loss, 0.000100 rate, 2.469925 seconds, 343072 images
Loaded: 0.000061 seconds

 10722: 0.013927, 0.012867 avg loss, 0.000100 rate, 2.570013 seconds, 343104 images
Loaded: 0.000040 seconds

 10723: 0.013271, 0.012908 avg loss, 0.000100 rate, 2.563766 seconds, 343136 images
Loaded: 0.000118 seconds

 10724: 0.012397, 0.012857 avg loss, 0.000100 rate, 2.560994 seconds, 343168 images
Loaded: 0.000076 seconds

 10725: 0.025411, 0.014112 avg loss, 0.000100 rate, 2.559807 seconds, 343200 images
Loaded: 0.000062 seconds

 10726: 0.013749, 0.014076 avg loss, 0.000100 rate, 2.560234 seconds, 343232 images
Loaded: 0.000042 seconds

 10727: 0.014997, 0.014168 avg loss, 0.000100 rate, 2.562454 seconds, 343264 images
Loaded: 0.000063 seconds

 10728: 0.009926, 0.013744 avg loss, 0.000100 rate, 2.567492 seconds, 343296 images
Loaded: 0.000067 seconds

 10729: 0.010315, 0.013401 avg loss, 0.000100 rate, 2.562385 seconds, 343328 images
Loaded: 0.000043 seconds

 10730: 0.033252, 0.015386 avg loss, 0.000100 rate, 2.562641 seconds, 343360 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000062 seconds

 10731: 0.019260, 0.015773 avg loss, 0.000100 rate, 1.854264 seconds, 343392 images
Loaded: 0.000042 seconds

 10732: 0.021779, 0.016374 avg loss, 0.000100 rate, 1.922499 seconds, 343424 images
Loaded: 0.000042 seconds

 10733: 0.016984, 0.016435 avg loss, 0.000100 rate, 1.932400 seconds, 343456 images
Loaded: 0.000043 seconds

 10734: 0.026295, 0.017421 avg loss, 0.000100 rate, 1.924709 seconds, 343488 images
Loaded: 0.000046 seconds

 10735: 0.016872, 0.017366 avg loss, 0.000100 rate, 1.918069 seconds, 343520 images
Loaded: 0.000042 seconds

 10736: 0.022217, 0.017851 avg loss, 0.000100 rate, 1.922409 seconds, 343552 images
Loaded: 0.000085 seconds

 10737: 0.014307, 0.017497 avg loss, 0.000100 rate, 1.925595 seconds, 343584 images
Loaded: 0.000082 seconds

 10738: 0.022106, 0.017958 avg loss, 0.000100 rate, 1.924562 seconds, 343616 images
Loaded: 0.000041 seconds

 10739: 0.034294, 0.019591 avg loss, 0.000100 rate, 1.920072 seconds, 343648 images
Loaded: 0.000039 seconds

 10740: 0.017641, 0.019396 avg loss, 0.000100 rate, 1.927531 seconds, 343680 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.094704 seconds

 10741: 0.019962, 0.019453 avg loss, 0.000100 rate, 1.846621 seconds, 343712 images
Loaded: 0.000044 seconds

 10742: 0.014605, 0.018968 avg loss, 0.000100 rate, 1.925834 seconds, 343744 images
Loaded: 0.000067 seconds

 10743: 0.016883, 0.018760 avg loss, 0.000100 rate, 1.924970 seconds, 343776 images
Loaded: 0.000063 seconds

 10744: 0.033228, 0.020206 avg loss, 0.000100 rate, 1.926649 seconds, 343808 images
Loaded: 0.000070 seconds

 10745: 0.011406, 0.019326 avg loss, 0.000100 rate, 1.921129 seconds, 343840 images
Loaded: 0.000056 seconds

 10746: 0.016697, 0.019063 avg loss, 0.000100 rate, 1.927357 seconds, 343872 images
Loaded: 0.000089 seconds

 10747: 0.032228, 0.020380 avg loss, 0.000100 rate, 1.925373 seconds, 343904 images
Loaded: 0.000071 seconds

 10748: 0.018755, 0.020217 avg loss, 0.000100 rate, 1.922134 seconds, 343936 images
Loaded: 0.000041 seconds

 10749: 0.010545, 0.019250 avg loss, 0.000100 rate, 1.924830 seconds, 343968 images
Loaded: 0.000086 seconds

 10750: 0.013106, 0.018636 avg loss, 0.000100 rate, 1.925872 seconds, 344000 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 10751: 0.012322, 0.018004 avg loss, 0.000100 rate, 2.081278 seconds, 344032 images
Loaded: 0.000042 seconds

 10752: 0.008545, 0.017059 avg loss, 0.000100 rate, 2.165541 seconds, 344064 images
Loaded: 0.000067 seconds

 10753: 0.010573, 0.016410 avg loss, 0.000100 rate, 2.167463 seconds, 344096 images
Loaded: 0.000068 seconds

 10754: 0.011987, 0.015968 avg loss, 0.000100 rate, 2.160004 seconds, 344128 images
Loaded: 0.000074 seconds

 10755: 0.015104, 0.015881 avg loss, 0.000100 rate, 2.165790 seconds, 344160 images
Loaded: 0.000067 seconds

 10756: 0.015846, 0.015878 avg loss, 0.000100 rate, 2.161164 seconds, 344192 images
Loaded: 0.000045 seconds

 10757: 0.014839, 0.015774 avg loss, 0.000100 rate, 2.168517 seconds, 344224 images
Loaded: 0.000041 seconds

 10758: 0.017009, 0.015897 avg loss, 0.000100 rate, 2.164956 seconds, 344256 images
Loaded: 0.000061 seconds

 10759: 0.017985, 0.016106 avg loss, 0.000100 rate, 2.173044 seconds, 344288 images
Loaded: 0.000051 seconds

 10760: 0.012608, 0.015756 avg loss, 0.000100 rate, 2.167303 seconds, 344320 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 10761: 0.013459, 0.015527 avg loss, 0.000100 rate, 4.160996 seconds, 344352 images
Loaded: 0.000041 seconds

 10762: 0.013965, 0.015370 avg loss, 0.000100 rate, 4.320337 seconds, 344384 images
Loaded: 0.000056 seconds

 10763: 0.014125, 0.015246 avg loss, 0.000100 rate, 4.307087 seconds, 344416 images
Loaded: 0.000075 seconds

 10764: 0.014277, 0.015149 avg loss, 0.000100 rate, 4.339994 seconds, 344448 images
Loaded: 0.000083 seconds

 10765: 0.012208, 0.014855 avg loss, 0.000100 rate, 4.322357 seconds, 344480 images
Loaded: 0.000043 seconds

 10766: 0.008713, 0.014241 avg loss, 0.000100 rate, 4.338886 seconds, 344512 images
Loaded: 0.000043 seconds

 10767: 0.015008, 0.014317 avg loss, 0.000100 rate, 4.331651 seconds, 344544 images
Loaded: 0.000036 seconds

 10768: 0.012195, 0.014105 avg loss, 0.000100 rate, 4.344821 seconds, 344576 images
Loaded: 0.000045 seconds

 10769: 0.009445, 0.013639 avg loss, 0.000100 rate, 4.339724 seconds, 344608 images
Loaded: 0.000090 seconds

 10770: 0.010267, 0.013302 avg loss, 0.000100 rate, 4.344145 seconds, 344640 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000053 seconds

 10771: 0.015579, 0.013530 avg loss, 0.000100 rate, 4.498183 seconds, 344672 images
Loaded: 0.000068 seconds

 10772: 0.018281, 0.014005 avg loss, 0.000100 rate, 4.662326 seconds, 344704 images
Loaded: 0.000051 seconds

 10773: 0.024302, 0.015034 avg loss, 0.000100 rate, 4.672149 seconds, 344736 images
Loaded: 0.000067 seconds

 10774: 0.010700, 0.014601 avg loss, 0.000100 rate, 4.660016 seconds, 344768 images
Loaded: 0.000043 seconds

 10775: 0.008963, 0.014037 avg loss, 0.000100 rate, 4.664322 seconds, 344800 images
Loaded: 0.000048 seconds

 10776: 0.013253, 0.013959 avg loss, 0.000100 rate, 4.681506 seconds, 344832 images
Loaded: 0.000068 seconds

 10777: 0.014334, 0.013996 avg loss, 0.000100 rate, 4.664824 seconds, 344864 images
Loaded: 0.000070 seconds

 10778: 0.015986, 0.014195 avg loss, 0.000100 rate, 4.667506 seconds, 344896 images
Loaded: 0.000073 seconds

 10779: 0.016840, 0.014460 avg loss, 0.000100 rate, 4.664368 seconds, 344928 images
Loaded: 0.000042 seconds

 10780: 0.010960, 0.014110 avg loss, 0.000100 rate, 4.672312 seconds, 344960 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.049633 seconds

 10781: 0.014836, 0.014182 avg loss, 0.000100 rate, 2.704295 seconds, 344992 images
Loaded: 0.000047 seconds

 10782: 0.010246, 0.013789 avg loss, 0.000100 rate, 2.813210 seconds, 345024 images
Loaded: 0.000044 seconds

 10783: 0.012184, 0.013628 avg loss, 0.000100 rate, 2.810236 seconds, 345056 images
Loaded: 0.000048 seconds

 10784: 0.010376, 0.013303 avg loss, 0.000100 rate, 2.811727 seconds, 345088 images
Loaded: 0.000064 seconds

 10785: 0.016750, 0.013648 avg loss, 0.000100 rate, 2.807420 seconds, 345120 images
Loaded: 0.000039 seconds

 10786: 0.011587, 0.013442 avg loss, 0.000100 rate, 2.805647 seconds, 345152 images
Loaded: 0.000041 seconds

 10787: 0.015229, 0.013620 avg loss, 0.000100 rate, 2.810259 seconds, 345184 images
Loaded: 0.000075 seconds

 10788: 0.011804, 0.013439 avg loss, 0.000100 rate, 2.812702 seconds, 345216 images
Loaded: 0.000043 seconds

 10789: 0.012268, 0.013322 avg loss, 0.000100 rate, 2.800953 seconds, 345248 images
Loaded: 0.000046 seconds

 10790: 0.014694, 0.013459 avg loss, 0.000100 rate, 2.809817 seconds, 345280 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 10791: 0.025683, 0.014681 avg loss, 0.000100 rate, 1.850383 seconds, 345312 images
Loaded: 0.000039 seconds

 10792: 0.033096, 0.016523 avg loss, 0.000100 rate, 1.925442 seconds, 345344 images
Loaded: 0.000061 seconds

 10793: 0.024658, 0.017336 avg loss, 0.000100 rate, 1.924257 seconds, 345376 images
Loaded: 0.000042 seconds

 10794: 0.016640, 0.017267 avg loss, 0.000100 rate, 1.919633 seconds, 345408 images
Loaded: 0.000087 seconds

 10795: 0.012370, 0.016777 avg loss, 0.000100 rate, 1.919063 seconds, 345440 images
Loaded: 0.000042 seconds

 10796: 0.012790, 0.016378 avg loss, 0.000100 rate, 1.919165 seconds, 345472 images
Loaded: 0.000051 seconds

 10797: 0.010760, 0.015817 avg loss, 0.000100 rate, 1.929128 seconds, 345504 images
Loaded: 0.000067 seconds

 10798: 0.010488, 0.015284 avg loss, 0.000100 rate, 1.924811 seconds, 345536 images
Loaded: 0.000032 seconds

 10799: 0.023368, 0.016092 avg loss, 0.000100 rate, 1.925163 seconds, 345568 images
Loaded: 0.000040 seconds

 10800: 0.011664, 0.015649 avg loss, 0.000010 rate, 1.918870 seconds, 345600 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 10801: 0.012288, 0.015313 avg loss, 0.000010 rate, 2.459976 seconds, 345632 images
Loaded: 0.000069 seconds

 10802: 0.010516, 0.014833 avg loss, 0.000010 rate, 2.551460 seconds, 345664 images
Loaded: 0.000043 seconds

 10803: 0.007376, 0.014088 avg loss, 0.000010 rate, 2.559864 seconds, 345696 images
Loaded: 0.000045 seconds

 10804: 0.009880, 0.013667 avg loss, 0.000010 rate, 2.554287 seconds, 345728 images
Loaded: 0.000044 seconds

 10805: 0.009823, 0.013283 avg loss, 0.000010 rate, 2.554227 seconds, 345760 images
Loaded: 0.000042 seconds

 10806: 0.014606, 0.013415 avg loss, 0.000010 rate, 2.563687 seconds, 345792 images
Loaded: 0.000069 seconds

 10807: 0.014391, 0.013513 avg loss, 0.000010 rate, 2.558932 seconds, 345824 images
Loaded: 0.000044 seconds

 10808: 0.019984, 0.014160 avg loss, 0.000010 rate, 2.565819 seconds, 345856 images
Loaded: 0.000044 seconds

 10809: 0.010699, 0.013814 avg loss, 0.000010 rate, 2.558887 seconds, 345888 images
Loaded: 0.000074 seconds

 10810: 0.011036, 0.013536 avg loss, 0.000010 rate, 2.561846 seconds, 345920 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 10811: 0.012285, 0.013411 avg loss, 0.000010 rate, 3.026993 seconds, 345952 images
Loaded: 0.000070 seconds

 10812: 0.012195, 0.013289 avg loss, 0.000010 rate, 3.144729 seconds, 345984 images
Loaded: 0.000038 seconds

 10813: 0.008172, 0.012778 avg loss, 0.000010 rate, 3.144196 seconds, 346016 images
Loaded: 0.000041 seconds

 10814: 0.011260, 0.012626 avg loss, 0.000010 rate, 3.146583 seconds, 346048 images
Loaded: 0.000043 seconds

 10815: 0.008588, 0.012222 avg loss, 0.000010 rate, 3.149559 seconds, 346080 images
Loaded: 0.000079 seconds

 10816: 0.012229, 0.012223 avg loss, 0.000010 rate, 3.157279 seconds, 346112 images
Loaded: 0.000054 seconds

 10817: 0.010174, 0.012018 avg loss, 0.000010 rate, 3.159999 seconds, 346144 images
Loaded: 0.000044 seconds

 10818: 0.011829, 0.011999 avg loss, 0.000010 rate, 3.149225 seconds, 346176 images
Loaded: 0.000041 seconds

 10819: 0.017010, 0.012500 avg loss, 0.000010 rate, 3.151990 seconds, 346208 images
Loaded: 0.000045 seconds

 10820: 0.010982, 0.012348 avg loss, 0.000010 rate, 3.146490 seconds, 346240 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.073604 seconds

 10821: 0.012917, 0.012405 avg loss, 0.000010 rate, 1.853117 seconds, 346272 images
Loaded: 0.000040 seconds

 10822: 0.039293, 0.015094 avg loss, 0.000010 rate, 1.931018 seconds, 346304 images
Loaded: 0.000079 seconds

 10823: 0.014976, 0.015082 avg loss, 0.000010 rate, 1.927213 seconds, 346336 images
Loaded: 0.000042 seconds

 10824: 0.017937, 0.015368 avg loss, 0.000010 rate, 1.928408 seconds, 346368 images
Loaded: 0.000068 seconds

 10825: 0.020247, 0.015855 avg loss, 0.000010 rate, 1.922728 seconds, 346400 images
Loaded: 0.000070 seconds

 10826: 0.019976, 0.016268 avg loss, 0.000010 rate, 1.925530 seconds, 346432 images
Loaded: 0.000035 seconds

 10827: 0.012328, 0.015874 avg loss, 0.000010 rate, 1.930621 seconds, 346464 images
Loaded: 0.000042 seconds

 10828: 0.019289, 0.016215 avg loss, 0.000010 rate, 1.944782 seconds, 346496 images
Loaded: 0.000045 seconds

 10829: 0.017051, 0.016299 avg loss, 0.000010 rate, 1.920969 seconds, 346528 images
Loaded: 0.000041 seconds

 10830: 0.020598, 0.016729 avg loss, 0.000010 rate, 1.925460 seconds, 346560 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 10831: 0.009354, 0.015991 avg loss, 0.000010 rate, 3.266743 seconds, 346592 images
Loaded: 0.000072 seconds

 10832: 0.011153, 0.015507 avg loss, 0.000010 rate, 3.382580 seconds, 346624 images
Loaded: 0.000046 seconds

 10833: 0.011842, 0.015141 avg loss, 0.000010 rate, 3.396735 seconds, 346656 images
Loaded: 0.000065 seconds

 10834: 0.006491, 0.014276 avg loss, 0.000010 rate, 3.386912 seconds, 346688 images
Loaded: 0.000047 seconds

 10835: 0.007619, 0.013610 avg loss, 0.000010 rate, 3.392599 seconds, 346720 images
Loaded: 0.000051 seconds

 10836: 0.016212, 0.013870 avg loss, 0.000010 rate, 3.396968 seconds, 346752 images
Loaded: 0.000069 seconds

 10837: 0.012113, 0.013695 avg loss, 0.000010 rate, 3.388512 seconds, 346784 images
Loaded: 0.000074 seconds

 10838: 0.012153, 0.013540 avg loss, 0.000010 rate, 3.393685 seconds, 346816 images
Loaded: 0.000074 seconds

 10839: 0.012955, 0.013482 avg loss, 0.000010 rate, 3.395632 seconds, 346848 images
Loaded: 0.000069 seconds

 10840: 0.010497, 0.013183 avg loss, 0.000010 rate, 3.390670 seconds, 346880 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.043076 seconds

 10841: 0.016203, 0.013485 avg loss, 0.000010 rate, 2.463343 seconds, 346912 images
Loaded: 0.000037 seconds

 10842: 0.010865, 0.013223 avg loss, 0.000010 rate, 2.562270 seconds, 346944 images
Loaded: 0.000034 seconds

 10843: 0.009536, 0.012855 avg loss, 0.000010 rate, 2.572756 seconds, 346976 images
Loaded: 0.000044 seconds

 10844: 0.010204, 0.012589 avg loss, 0.000010 rate, 2.558278 seconds, 347008 images
Loaded: 0.000081 seconds

 10845: 0.010769, 0.012407 avg loss, 0.000010 rate, 2.567193 seconds, 347040 images
Loaded: 0.000074 seconds

 10846: 0.019597, 0.013126 avg loss, 0.000010 rate, 2.557099 seconds, 347072 images
Loaded: 0.000042 seconds

 10847: 0.015103, 0.013324 avg loss, 0.000010 rate, 2.565599 seconds, 347104 images
Loaded: 0.000080 seconds

 10848: 0.008605, 0.012852 avg loss, 0.000010 rate, 2.572477 seconds, 347136 images
Loaded: 0.000043 seconds

 10849: 0.013778, 0.012945 avg loss, 0.000010 rate, 2.559868 seconds, 347168 images
Loaded: 0.000042 seconds

 10850: 0.010162, 0.012666 avg loss, 0.000010 rate, 2.558069 seconds, 347200 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 10851: 0.014075, 0.012807 avg loss, 0.000010 rate, 4.518277 seconds, 347232 images
Loaded: 0.000065 seconds

 10852: 0.013512, 0.012878 avg loss, 0.000010 rate, 4.666869 seconds, 347264 images
Loaded: 0.000049 seconds

 10853: 0.011484, 0.012738 avg loss, 0.000010 rate, 4.671056 seconds, 347296 images
Loaded: 0.000045 seconds

 10854: 0.012051, 0.012670 avg loss, 0.000010 rate, 4.671054 seconds, 347328 images
Loaded: 0.000046 seconds

 10855: 0.010399, 0.012443 avg loss, 0.000010 rate, 4.673489 seconds, 347360 images
Loaded: 0.000071 seconds

 10856: 0.018729, 0.013071 avg loss, 0.000010 rate, 4.677997 seconds, 347392 images
Loaded: 0.000045 seconds

 10857: 0.013537, 0.013118 avg loss, 0.000010 rate, 4.672414 seconds, 347424 images
Loaded: 0.000083 seconds

 10858: 0.017287, 0.013535 avg loss, 0.000010 rate, 4.695637 seconds, 347456 images
Loaded: 0.000043 seconds

 10859: 0.016238, 0.013805 avg loss, 0.000010 rate, 4.678019 seconds, 347488 images
Loaded: 0.000067 seconds

 10860: 0.024749, 0.014899 avg loss, 0.000010 rate, 4.686713 seconds, 347520 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 10861: 0.013352, 0.014745 avg loss, 0.000010 rate, 2.085900 seconds, 347552 images
Loaded: 0.000069 seconds

 10862: 0.010485, 0.014319 avg loss, 0.000010 rate, 2.174513 seconds, 347584 images
Loaded: 0.000046 seconds

 10863: 0.011121, 0.013999 avg loss, 0.000010 rate, 2.172961 seconds, 347616 images
Loaded: 0.000074 seconds

 10864: 0.017307, 0.014330 avg loss, 0.000010 rate, 2.170899 seconds, 347648 images
Loaded: 0.000048 seconds

 10865: 0.009708, 0.013868 avg loss, 0.000010 rate, 2.178130 seconds, 347680 images
Loaded: 0.000051 seconds

 10866: 0.013722, 0.013853 avg loss, 0.000010 rate, 2.167989 seconds, 347712 images
Loaded: 0.000065 seconds

 10867: 0.013482, 0.013816 avg loss, 0.000010 rate, 2.171028 seconds, 347744 images
Loaded: 0.000049 seconds

 10868: 0.011744, 0.013609 avg loss, 0.000010 rate, 2.174266 seconds, 347776 images
Loaded: 0.000039 seconds

 10869: 0.011799, 0.013428 avg loss, 0.000010 rate, 2.170983 seconds, 347808 images
Loaded: 0.000046 seconds

 10870: 0.015720, 0.013657 avg loss, 0.000010 rate, 2.175118 seconds, 347840 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 10871: 0.008153, 0.013107 avg loss, 0.000010 rate, 2.694329 seconds, 347872 images
Loaded: 0.000066 seconds

 10872: 0.013085, 0.013104 avg loss, 0.000010 rate, 2.804422 seconds, 347904 images
Loaded: 0.000049 seconds

 10873: 0.021260, 0.013920 avg loss, 0.000010 rate, 2.806602 seconds, 347936 images
Loaded: 0.000062 seconds

 10874: 0.007091, 0.013237 avg loss, 0.000010 rate, 2.804197 seconds, 347968 images
Loaded: 0.000082 seconds

 10875: 0.010493, 0.012963 avg loss, 0.000010 rate, 2.806938 seconds, 348000 images
Loaded: 0.000044 seconds

 10876: 0.018008, 0.013467 avg loss, 0.000010 rate, 2.805152 seconds, 348032 images
Loaded: 0.000067 seconds

 10877: 0.008932, 0.013014 avg loss, 0.000010 rate, 2.803705 seconds, 348064 images
Loaded: 0.000047 seconds

 10878: 0.010890, 0.012801 avg loss, 0.000010 rate, 2.799794 seconds, 348096 images
Loaded: 0.000044 seconds

 10879: 0.009666, 0.012488 avg loss, 0.000010 rate, 2.803229 seconds, 348128 images
Loaded: 0.000043 seconds

 10880: 0.008360, 0.012075 avg loss, 0.000010 rate, 2.804782 seconds, 348160 images
Resizing
608 x 608 
 try to allocate additional workspace_size = 108.69 MB 
 CUDA allocate done! 
Loaded: 0.000060 seconds

 10881: 0.029998, 0.013867 avg loss, 0.000010 rate, 4.906250 seconds, 348192 images
Loaded: 0.000039 seconds

 10882: 0.018716, 0.014352 avg loss, 0.000010 rate, 5.088035 seconds, 348224 images
Loaded: 0.000042 seconds

 10883: 0.012776, 0.014195 avg loss, 0.000010 rate, 5.111255 seconds, 348256 images
Loaded: 0.000070 seconds

 10884: 0.018601, 0.014635 avg loss, 0.000010 rate, 5.110323 seconds, 348288 images
Loaded: 0.000080 seconds

 10885: 0.016016, 0.014773 avg loss, 0.000010 rate, 5.108197 seconds, 348320 images
Loaded: 0.000049 seconds

 10886: 0.023365, 0.015632 avg loss, 0.000010 rate, 5.116342 seconds, 348352 images
Loaded: 0.000079 seconds

 10887: 0.019933, 0.016062 avg loss, 0.000010 rate, 5.109579 seconds, 348384 images
Loaded: 0.000079 seconds

 10888: 0.016307, 0.016087 avg loss, 0.000010 rate, 5.123074 seconds, 348416 images
Loaded: 0.000073 seconds

 10889: 0.020078, 0.016486 avg loss, 0.000010 rate, 5.110285 seconds, 348448 images
Loaded: 0.000066 seconds

 10890: 0.019589, 0.016796 avg loss, 0.000010 rate, 5.119519 seconds, 348480 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 10891: 0.012207, 0.016337 avg loss, 0.000010 rate, 3.268502 seconds, 348512 images
Loaded: 0.000043 seconds

 10892: 0.010771, 0.015781 avg loss, 0.000010 rate, 3.390056 seconds, 348544 images
Loaded: 0.000044 seconds

 10893: 0.013849, 0.015588 avg loss, 0.000010 rate, 3.392468 seconds, 348576 images
Loaded: 0.000045 seconds

 10894: 0.007784, 0.014807 avg loss, 0.000010 rate, 3.389686 seconds, 348608 images
Loaded: 0.000072 seconds

 10895: 0.028588, 0.016185 avg loss, 0.000010 rate, 3.388186 seconds, 348640 images
Loaded: 0.000076 seconds

 10896: 0.012926, 0.015859 avg loss, 0.000010 rate, 3.395708 seconds, 348672 images
Loaded: 0.000040 seconds

 10897: 0.016092, 0.015883 avg loss, 0.000010 rate, 3.390521 seconds, 348704 images
Loaded: 0.000054 seconds

 10898: 0.009550, 0.015249 avg loss, 0.000010 rate, 3.401624 seconds, 348736 images
Loaded: 0.000068 seconds

 10899: 0.010670, 0.014791 avg loss, 0.000010 rate, 3.396614 seconds, 348768 images
Loaded: 0.000058 seconds

 10900: 0.011811, 0.014493 avg loss, 0.000010 rate, 3.401249 seconds, 348800 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.345476 seconds

 10901: 0.012359, 0.014280 avg loss, 0.000010 rate, 2.705113 seconds, 348832 images
Loaded: 0.000041 seconds

 10902: 0.011893, 0.014041 avg loss, 0.000010 rate, 2.810365 seconds, 348864 images
Loaded: 0.000079 seconds

 10903: 0.010230, 0.013660 avg loss, 0.000010 rate, 2.808231 seconds, 348896 images
Loaded: 0.000036 seconds

 10904: 0.013163, 0.013610 avg loss, 0.000010 rate, 2.812006 seconds, 348928 images
Loaded: 0.000042 seconds

 10905: 0.008065, 0.013056 avg loss, 0.000010 rate, 2.807664 seconds, 348960 images
Loaded: 0.000043 seconds

 10906: 0.010714, 0.012822 avg loss, 0.000010 rate, 2.801891 seconds, 348992 images
Loaded: 0.000043 seconds

 10907: 0.011455, 0.012685 avg loss, 0.000010 rate, 2.799394 seconds, 349024 images
Loaded: 0.000071 seconds

 10908: 0.012709, 0.012688 avg loss, 0.000010 rate, 2.816074 seconds, 349056 images
Loaded: 0.000066 seconds

 10909: 0.010506, 0.012469 avg loss, 0.000010 rate, 2.815034 seconds, 349088 images
Loaded: 0.000038 seconds

 10910: 0.019953, 0.013218 avg loss, 0.000010 rate, 2.802856 seconds, 349120 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000070 seconds

 10911: 0.014701, 0.013366 avg loss, 0.000010 rate, 1.846247 seconds, 349152 images
Loaded: 0.000070 seconds

 10912: 0.016694, 0.013699 avg loss, 0.000010 rate, 1.924302 seconds, 349184 images
Loaded: 0.000039 seconds

 10913: 0.022136, 0.014543 avg loss, 0.000010 rate, 1.919548 seconds, 349216 images
Loaded: 0.000087 seconds

 10914: 0.022679, 0.015356 avg loss, 0.000010 rate, 1.923116 seconds, 349248 images
Loaded: 0.000069 seconds

 10915: 0.011235, 0.014944 avg loss, 0.000010 rate, 1.920115 seconds, 349280 images
Loaded: 0.000042 seconds

 10916: 0.017799, 0.015230 avg loss, 0.000010 rate, 1.921308 seconds, 349312 images
Loaded: 0.000062 seconds

 10917: 0.018449, 0.015552 avg loss, 0.000010 rate, 1.921243 seconds, 349344 images
Loaded: 0.000043 seconds

 10918: 0.022854, 0.016282 avg loss, 0.000010 rate, 1.929986 seconds, 349376 images
Loaded: 0.000044 seconds

 10919: 0.022286, 0.016882 avg loss, 0.000010 rate, 1.927132 seconds, 349408 images
Loaded: 0.000076 seconds

 10920: 0.011217, 0.016316 avg loss, 0.000010 rate, 1.921869 seconds, 349440 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 10921: 0.020387, 0.016723 avg loss, 0.000010 rate, 4.172166 seconds, 349472 images
Loaded: 0.000051 seconds

 10922: 0.006603, 0.015711 avg loss, 0.000010 rate, 4.333136 seconds, 349504 images
Loaded: 0.000072 seconds

 10923: 0.011396, 0.015279 avg loss, 0.000010 rate, 4.319506 seconds, 349536 images
Loaded: 0.000042 seconds

 10924: 0.017365, 0.015488 avg loss, 0.000010 rate, 4.331846 seconds, 349568 images
Loaded: 0.000045 seconds

 10925: 0.010366, 0.014976 avg loss, 0.000010 rate, 4.326033 seconds, 349600 images
Loaded: 0.000045 seconds

 10926: 0.012772, 0.014755 avg loss, 0.000010 rate, 4.337671 seconds, 349632 images
Loaded: 0.000045 seconds

 10927: 0.012874, 0.014567 avg loss, 0.000010 rate, 4.337076 seconds, 349664 images
Loaded: 0.000067 seconds

 10928: 0.012800, 0.014391 avg loss, 0.000010 rate, 4.335822 seconds, 349696 images
Loaded: 0.000036 seconds

 10929: 0.011286, 0.014080 avg loss, 0.000010 rate, 4.344952 seconds, 349728 images
Loaded: 0.000038 seconds

 10930: 0.007494, 0.013422 avg loss, 0.000010 rate, 4.354351 seconds, 349760 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000064 seconds

 10931: 0.013370, 0.013416 avg loss, 0.000010 rate, 1.854741 seconds, 349792 images
Loaded: 0.000042 seconds

 10932: 0.045480, 0.016623 avg loss, 0.000010 rate, 1.929808 seconds, 349824 images
Loaded: 0.000043 seconds

 10933: 0.017410, 0.016701 avg loss, 0.000010 rate, 1.931420 seconds, 349856 images
Loaded: 0.000061 seconds

 10934: 0.018257, 0.016857 avg loss, 0.000010 rate, 1.924872 seconds, 349888 images
Loaded: 0.000040 seconds

 10935: 0.020838, 0.017255 avg loss, 0.000010 rate, 1.926996 seconds, 349920 images
Loaded: 0.000044 seconds

 10936: 0.018862, 0.017416 avg loss, 0.000010 rate, 1.932275 seconds, 349952 images
Loaded: 0.000068 seconds

 10937: 0.019966, 0.017671 avg loss, 0.000010 rate, 1.919077 seconds, 349984 images
Loaded: 0.000041 seconds

 10938: 0.011225, 0.017026 avg loss, 0.000010 rate, 1.923842 seconds, 350016 images
Loaded: 0.000074 seconds

 10939: 0.013888, 0.016712 avg loss, 0.000010 rate, 1.924462 seconds, 350048 images
Loaded: 0.000060 seconds

 10940: 0.016830, 0.016724 avg loss, 0.000010 rate, 1.926273 seconds, 350080 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.083938 seconds

 10941: 0.015184, 0.016570 avg loss, 0.000010 rate, 1.848617 seconds, 350112 images
Loaded: 0.000038 seconds

 10942: 0.009806, 0.015894 avg loss, 0.000010 rate, 1.923902 seconds, 350144 images
Loaded: 0.000041 seconds

 10943: 0.014213, 0.015726 avg loss, 0.000010 rate, 1.926488 seconds, 350176 images
Loaded: 0.000068 seconds

 10944: 0.028108, 0.016964 avg loss, 0.000010 rate, 1.925434 seconds, 350208 images
Loaded: 0.000041 seconds

 10945: 0.013766, 0.016644 avg loss, 0.000010 rate, 1.921793 seconds, 350240 images
Loaded: 0.000066 seconds

 10946: 0.026761, 0.017656 avg loss, 0.000010 rate, 1.919935 seconds, 350272 images
Loaded: 0.000036 seconds

 10947: 0.016414, 0.017532 avg loss, 0.000010 rate, 1.927766 seconds, 350304 images
Loaded: 0.000065 seconds

 10948: 0.012556, 0.017034 avg loss, 0.000010 rate, 1.919003 seconds, 350336 images
Loaded: 0.000043 seconds

 10949: 0.023189, 0.017650 avg loss, 0.000010 rate, 1.922039 seconds, 350368 images
Loaded: 0.000045 seconds

 10950: 0.015923, 0.017477 avg loss, 0.000010 rate, 1.927162 seconds, 350400 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.083724 seconds

 10951: 0.019021, 0.017631 avg loss, 0.000010 rate, 1.845057 seconds, 350432 images
Loaded: 0.000036 seconds

 10952: 0.020036, 0.017872 avg loss, 0.000010 rate, 1.924871 seconds, 350464 images
Loaded: 0.000073 seconds

 10953: 0.047845, 0.020869 avg loss, 0.000010 rate, 1.922972 seconds, 350496 images
Loaded: 0.000079 seconds

 10954: 0.013100, 0.020092 avg loss, 0.000010 rate, 1.926039 seconds, 350528 images
Loaded: 0.000040 seconds

 10955: 0.012988, 0.019382 avg loss, 0.000010 rate, 1.930075 seconds, 350560 images
Loaded: 0.000064 seconds

 10956: 0.017417, 0.019185 avg loss, 0.000010 rate, 1.924308 seconds, 350592 images
Loaded: 0.000086 seconds

 10957: 0.017333, 0.019000 avg loss, 0.000010 rate, 1.927623 seconds, 350624 images
Loaded: 0.000066 seconds

 10958: 0.014220, 0.018522 avg loss, 0.000010 rate, 1.921521 seconds, 350656 images
Loaded: 0.000068 seconds

 10959: 0.043714, 0.021041 avg loss, 0.000010 rate, 1.917391 seconds, 350688 images
Loaded: 0.000056 seconds

 10960: 0.015083, 0.020446 avg loss, 0.000010 rate, 1.926781 seconds, 350720 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000038 seconds

 10961: 0.009281, 0.019329 avg loss, 0.000010 rate, 3.254922 seconds, 350752 images
Loaded: 0.000042 seconds

 10962: 0.009707, 0.018367 avg loss, 0.000010 rate, 3.381722 seconds, 350784 images
Loaded: 0.000060 seconds

 10963: 0.015076, 0.018038 avg loss, 0.000010 rate, 3.378627 seconds, 350816 images
Loaded: 0.000047 seconds

 10964: 0.007568, 0.016991 avg loss, 0.000010 rate, 3.384905 seconds, 350848 images
Loaded: 0.000071 seconds

 10965: 0.008922, 0.016184 avg loss, 0.000010 rate, 3.363446 seconds, 350880 images
Loaded: 0.000050 seconds

 10966: 0.009419, 0.015507 avg loss, 0.000010 rate, 3.387449 seconds, 350912 images
Loaded: 0.000042 seconds

 10967: 0.013960, 0.015353 avg loss, 0.000010 rate, 3.379557 seconds, 350944 images
Loaded: 0.000043 seconds

 10968: 0.011765, 0.014994 avg loss, 0.000010 rate, 3.395328 seconds, 350976 images
Loaded: 0.000038 seconds

 10969: 0.011200, 0.014615 avg loss, 0.000010 rate, 3.377652 seconds, 351008 images
Loaded: 0.000043 seconds

 10970: 0.009596, 0.014113 avg loss, 0.000010 rate, 3.391118 seconds, 351040 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.036340 seconds

 10971: 0.013072, 0.014009 avg loss, 0.000010 rate, 2.468421 seconds, 351072 images
Loaded: 0.000052 seconds

 10972: 0.010858, 0.013694 avg loss, 0.000010 rate, 2.562718 seconds, 351104 images
Loaded: 0.000044 seconds

 10973: 0.012482, 0.013572 avg loss, 0.000010 rate, 2.558792 seconds, 351136 images
Loaded: 0.000043 seconds

 10974: 0.009694, 0.013185 avg loss, 0.000010 rate, 2.562526 seconds, 351168 images
Loaded: 0.000040 seconds

 10975: 0.013462, 0.013212 avg loss, 0.000010 rate, 2.558958 seconds, 351200 images
Loaded: 0.000069 seconds

 10976: 0.009720, 0.012863 avg loss, 0.000010 rate, 2.565924 seconds, 351232 images
Loaded: 0.000068 seconds

 10977: 0.011964, 0.012773 avg loss, 0.000010 rate, 2.565499 seconds, 351264 images
Loaded: 0.000042 seconds

 10978: 0.013189, 0.012815 avg loss, 0.000010 rate, 2.561575 seconds, 351296 images
Loaded: 0.000049 seconds

 10979: 0.008063, 0.012340 avg loss, 0.000010 rate, 2.560097 seconds, 351328 images
Loaded: 0.000065 seconds

 10980: 0.010033, 0.012109 avg loss, 0.000010 rate, 2.568939 seconds, 351360 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 10981: 0.008859, 0.011784 avg loss, 0.000010 rate, 4.496542 seconds, 351392 images
Loaded: 0.000074 seconds

 10982: 0.024251, 0.013031 avg loss, 0.000010 rate, 4.665354 seconds, 351424 images
Loaded: 0.000067 seconds

 10983: 0.011704, 0.012898 avg loss, 0.000010 rate, 4.673307 seconds, 351456 images
Loaded: 0.000046 seconds

 10984: 0.012054, 0.012814 avg loss, 0.000010 rate, 4.661841 seconds, 351488 images
Loaded: 0.000067 seconds

 10985: 0.011871, 0.012719 avg loss, 0.000010 rate, 4.681887 seconds, 351520 images
Loaded: 0.000115 seconds

 10986: 0.007975, 0.012245 avg loss, 0.000010 rate, 4.691837 seconds, 351552 images
Loaded: 0.000041 seconds

 10987: 0.012105, 0.012231 avg loss, 0.000010 rate, 4.679271 seconds, 351584 images
Loaded: 0.000067 seconds

 10988: 0.013597, 0.012368 avg loss, 0.000010 rate, 4.681587 seconds, 351616 images
Loaded: 0.000044 seconds

 10989: 0.011074, 0.012238 avg loss, 0.000010 rate, 4.663171 seconds, 351648 images
Loaded: 0.000049 seconds

 10990: 0.015808, 0.012595 avg loss, 0.000010 rate, 4.665232 seconds, 351680 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.026234 seconds

 10991: 0.009317, 0.012267 avg loss, 0.000010 rate, 2.471926 seconds, 351712 images
Loaded: 0.000042 seconds

 10992: 0.010856, 0.012126 avg loss, 0.000010 rate, 2.562628 seconds, 351744 images
Loaded: 0.000082 seconds

 10993: 0.010421, 0.011956 avg loss, 0.000010 rate, 2.555355 seconds, 351776 images
Loaded: 0.000055 seconds

 10994: 0.013462, 0.012106 avg loss, 0.000010 rate, 2.565348 seconds, 351808 images
Loaded: 0.000044 seconds

 10995: 0.014429, 0.012339 avg loss, 0.000010 rate, 2.558226 seconds, 351840 images
Loaded: 0.000044 seconds

 10996: 0.019529, 0.013058 avg loss, 0.000010 rate, 2.564492 seconds, 351872 images
Loaded: 0.000061 seconds

 10997: 0.010495, 0.012801 avg loss, 0.000010 rate, 2.564335 seconds, 351904 images
Loaded: 0.000040 seconds

 10998: 0.014149, 0.012936 avg loss, 0.000010 rate, 2.553841 seconds, 351936 images
Loaded: 0.000088 seconds

 10999: 0.008340, 0.012477 avg loss, 0.000010 rate, 2.566092 seconds, 351968 images
Loaded: 0.000043 seconds

 11000: 0.013864, 0.012615 avg loss, 0.000010 rate, 2.563089 seconds, 352000 images
Saving weights to backup/custom-train-yolo_11000.weights
Saving weights to backup/custom-train-yolo_last.weights
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000066 seconds

 11001: 0.007893, 0.012143 avg loss, 0.000010 rate, 3.270155 seconds, 352032 images
Loaded: 0.000065 seconds

 11002: 0.012581, 0.012187 avg loss, 0.000010 rate, 3.385488 seconds, 352064 images
Loaded: 0.000035 seconds

 11003: 0.010405, 0.012009 avg loss, 0.000010 rate, 3.384543 seconds, 352096 images
Loaded: 0.000043 seconds

 11004: 0.012184, 0.012026 avg loss, 0.000010 rate, 3.386198 seconds, 352128 images
Loaded: 0.000041 seconds

 11005: 0.011615, 0.011985 avg loss, 0.000010 rate, 3.384628 seconds, 352160 images
Loaded: 0.000059 seconds

 11006: 0.011598, 0.011946 avg loss, 0.000010 rate, 3.385635 seconds, 352192 images
Loaded: 0.000061 seconds

 11007: 0.009899, 0.011742 avg loss, 0.000010 rate, 3.390545 seconds, 352224 images
Loaded: 0.000042 seconds

 11008: 0.015318, 0.012099 avg loss, 0.000010 rate, 3.388750 seconds, 352256 images
Loaded: 0.000039 seconds

 11009: 0.013982, 0.012288 avg loss, 0.000010 rate, 3.389001 seconds, 352288 images
Loaded: 0.000043 seconds

 11010: 0.009844, 0.012043 avg loss, 0.000010 rate, 3.396778 seconds, 352320 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 11011: 0.011039, 0.011943 avg loss, 0.000010 rate, 4.196287 seconds, 352352 images
Loaded: 0.000041 seconds

 11012: 0.008335, 0.011582 avg loss, 0.000010 rate, 4.333149 seconds, 352384 images
Loaded: 0.000042 seconds

 11013: 0.010886, 0.011512 avg loss, 0.000010 rate, 4.333592 seconds, 352416 images
Loaded: 0.000066 seconds

 11014: 0.009989, 0.011360 avg loss, 0.000010 rate, 4.345341 seconds, 352448 images
Loaded: 0.000044 seconds

 11015: 0.007715, 0.010996 avg loss, 0.000010 rate, 4.342762 seconds, 352480 images
Loaded: 0.000044 seconds

 11016: 0.009776, 0.010874 avg loss, 0.000010 rate, 4.339642 seconds, 352512 images
Loaded: 0.000078 seconds

 11017: 0.012621, 0.011048 avg loss, 0.000010 rate, 4.347339 seconds, 352544 images
Loaded: 0.000045 seconds

 11018: 0.010736, 0.011017 avg loss, 0.000010 rate, 4.352450 seconds, 352576 images
Loaded: 0.000071 seconds

 11019: 0.009450, 0.010860 avg loss, 0.000010 rate, 4.345146 seconds, 352608 images
Loaded: 0.000047 seconds

 11020: 0.014567, 0.011231 avg loss, 0.000010 rate, 4.340152 seconds, 352640 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000033 seconds

 11021: 0.008889, 0.010997 avg loss, 0.000010 rate, 3.020266 seconds, 352672 images
Loaded: 0.000042 seconds

 11022: 0.012093, 0.011106 avg loss, 0.000010 rate, 3.146110 seconds, 352704 images
Loaded: 0.000047 seconds

 11023: 0.014699, 0.011466 avg loss, 0.000010 rate, 3.159738 seconds, 352736 images
Loaded: 0.000043 seconds

 11024: 0.010313, 0.011350 avg loss, 0.000010 rate, 3.147572 seconds, 352768 images
Loaded: 0.000045 seconds

 11025: 0.013205, 0.011536 avg loss, 0.000010 rate, 3.142211 seconds, 352800 images
Loaded: 0.000046 seconds

 11026: 0.018723, 0.012255 avg loss, 0.000010 rate, 3.143554 seconds, 352832 images
Loaded: 0.000061 seconds

 11027: 0.014040, 0.012433 avg loss, 0.000010 rate, 3.142180 seconds, 352864 images
Loaded: 0.000047 seconds

 11028: 0.020260, 0.013216 avg loss, 0.000010 rate, 3.158139 seconds, 352896 images
Loaded: 0.000059 seconds

 11029: 0.008402, 0.012734 avg loss, 0.000010 rate, 3.136768 seconds, 352928 images
Loaded: 0.000042 seconds

 11030: 0.014523, 0.012913 avg loss, 0.000010 rate, 3.146766 seconds, 352960 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.021764 seconds

 11031: 0.009589, 0.012581 avg loss, 0.000010 rate, 3.025250 seconds, 352992 images
Loaded: 0.000052 seconds

 11032: 0.016797, 0.013002 avg loss, 0.000010 rate, 3.142083 seconds, 353024 images
Loaded: 0.000042 seconds

 11033: 0.011657, 0.012868 avg loss, 0.000010 rate, 3.139241 seconds, 353056 images
Loaded: 0.000042 seconds

 11034: 0.010225, 0.012604 avg loss, 0.000010 rate, 3.140836 seconds, 353088 images
Loaded: 0.000066 seconds

 11035: 0.007881, 0.012131 avg loss, 0.000010 rate, 3.131432 seconds, 353120 images
Loaded: 0.000036 seconds

 11036: 0.014169, 0.012335 avg loss, 0.000010 rate, 3.128444 seconds, 353152 images
Loaded: 0.000045 seconds

 11037: 0.014636, 0.012565 avg loss, 0.000010 rate, 3.137388 seconds, 353184 images
Loaded: 0.000067 seconds

 11038: 0.011498, 0.012459 avg loss, 0.000010 rate, 3.143512 seconds, 353216 images
Loaded: 0.000044 seconds

 11039: 0.013286, 0.012541 avg loss, 0.000010 rate, 3.139284 seconds, 353248 images
Loaded: 0.000043 seconds

 11040: 0.011371, 0.012424 avg loss, 0.000010 rate, 3.136484 seconds, 353280 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.004494 seconds

 11041: 0.013367, 0.012519 avg loss, 0.000010 rate, 3.030443 seconds, 353312 images
Loaded: 0.000043 seconds

 11042: 0.010184, 0.012285 avg loss, 0.000010 rate, 3.136084 seconds, 353344 images
Loaded: 0.000067 seconds

 11043: 0.011984, 0.012255 avg loss, 0.000010 rate, 3.131369 seconds, 353376 images
Loaded: 0.000043 seconds

 11044: 0.011141, 0.012144 avg loss, 0.000010 rate, 3.139215 seconds, 353408 images
Loaded: 0.000042 seconds

 11045: 0.009612, 0.011890 avg loss, 0.000010 rate, 3.141162 seconds, 353440 images
Loaded: 0.000042 seconds

 11046: 0.012744, 0.011976 avg loss, 0.000010 rate, 3.132935 seconds, 353472 images
Loaded: 0.000035 seconds

 11047: 0.008393, 0.011618 avg loss, 0.000010 rate, 3.138109 seconds, 353504 images
Loaded: 0.000045 seconds

 11048: 0.009273, 0.011383 avg loss, 0.000010 rate, 3.128570 seconds, 353536 images
Loaded: 0.000041 seconds

 11049: 0.010075, 0.011252 avg loss, 0.000010 rate, 3.136737 seconds, 353568 images
Loaded: 0.000048 seconds

 11050: 0.014269, 0.011554 avg loss, 0.000010 rate, 3.124840 seconds, 353600 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11051: 0.014079, 0.011806 avg loss, 0.000010 rate, 4.483282 seconds, 353632 images
Loaded: 0.000042 seconds

 11052: 0.023469, 0.012973 avg loss, 0.000010 rate, 4.663948 seconds, 353664 images
Loaded: 0.000079 seconds

 11053: 0.022565, 0.013932 avg loss, 0.000010 rate, 4.662852 seconds, 353696 images
Loaded: 0.000063 seconds

 11054: 0.011687, 0.013707 avg loss, 0.000010 rate, 4.661740 seconds, 353728 images
Loaded: 0.000080 seconds

 11055: 0.015267, 0.013863 avg loss, 0.000010 rate, 4.667266 seconds, 353760 images
Loaded: 0.000047 seconds

 11056: 0.020452, 0.014522 avg loss, 0.000010 rate, 4.667472 seconds, 353792 images
Loaded: 0.000075 seconds

 11057: 0.014402, 0.014510 avg loss, 0.000010 rate, 4.662734 seconds, 353824 images
Loaded: 0.000069 seconds

 11058: 0.010020, 0.014061 avg loss, 0.000010 rate, 4.667457 seconds, 353856 images
Loaded: 0.000076 seconds

 11059: 0.016229, 0.014278 avg loss, 0.000010 rate, 4.674688 seconds, 353888 images
Loaded: 0.000044 seconds

 11060: 0.009322, 0.013782 avg loss, 0.000010 rate, 4.674026 seconds, 353920 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000034 seconds

 11061: 0.010531, 0.013457 avg loss, 0.000010 rate, 3.271343 seconds, 353952 images
Loaded: 0.000042 seconds

 11062: 0.011950, 0.013307 avg loss, 0.000010 rate, 3.386311 seconds, 353984 images
Loaded: 0.000068 seconds

 11063: 0.009654, 0.012941 avg loss, 0.000010 rate, 3.392135 seconds, 354016 images
Loaded: 0.000053 seconds

 11064: 0.017244, 0.013372 avg loss, 0.000010 rate, 3.387804 seconds, 354048 images
Loaded: 0.000067 seconds

 11065: 0.020869, 0.014121 avg loss, 0.000010 rate, 3.389108 seconds, 354080 images
Loaded: 0.000047 seconds

 11066: 0.013234, 0.014033 avg loss, 0.000010 rate, 3.400831 seconds, 354112 images
Loaded: 0.000067 seconds

 11067: 0.015653, 0.014195 avg loss, 0.000010 rate, 3.392048 seconds, 354144 images
Loaded: 0.000043 seconds

 11068: 0.012237, 0.013999 avg loss, 0.000010 rate, 3.396755 seconds, 354176 images
Loaded: 0.000042 seconds

 11069: 0.011788, 0.013778 avg loss, 0.000010 rate, 3.389809 seconds, 354208 images
Loaded: 0.000061 seconds

 11070: 0.011484, 0.013548 avg loss, 0.000010 rate, 3.398943 seconds, 354240 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.034793 seconds

 11071: 0.013501, 0.013544 avg loss, 0.000010 rate, 2.466129 seconds, 354272 images
Loaded: 0.000042 seconds

 11072: 0.008192, 0.013008 avg loss, 0.000010 rate, 2.565117 seconds, 354304 images
Loaded: 0.000041 seconds

 11073: 0.012597, 0.012967 avg loss, 0.000010 rate, 2.567618 seconds, 354336 images
Loaded: 0.000043 seconds

 11074: 0.016102, 0.013281 avg loss, 0.000010 rate, 2.562240 seconds, 354368 images
Loaded: 0.000041 seconds

 11075: 0.014670, 0.013420 avg loss, 0.000010 rate, 2.561752 seconds, 354400 images
Loaded: 0.000080 seconds

 11076: 0.019173, 0.013995 avg loss, 0.000010 rate, 2.569390 seconds, 354432 images
Loaded: 0.000064 seconds

 11077: 0.012478, 0.013843 avg loss, 0.000010 rate, 2.561063 seconds, 354464 images
Loaded: 0.000055 seconds

 11078: 0.009744, 0.013433 avg loss, 0.000010 rate, 2.561156 seconds, 354496 images
Loaded: 0.000066 seconds

 11079: 0.009523, 0.013042 avg loss, 0.000010 rate, 2.561395 seconds, 354528 images
Loaded: 0.000059 seconds

 11080: 0.037106, 0.015449 avg loss, 0.000010 rate, 2.564989 seconds, 354560 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.037306 seconds

 11081: 0.011712, 0.015075 avg loss, 0.000010 rate, 2.461810 seconds, 354592 images
Loaded: 0.000067 seconds

 11082: 0.009365, 0.014504 avg loss, 0.000010 rate, 2.561331 seconds, 354624 images
Loaded: 0.000059 seconds

 11083: 0.009799, 0.014034 avg loss, 0.000010 rate, 2.564699 seconds, 354656 images
Loaded: 0.000070 seconds

 11084: 0.016661, 0.014296 avg loss, 0.000010 rate, 2.558773 seconds, 354688 images
Loaded: 0.000070 seconds

 11085: 0.010182, 0.013885 avg loss, 0.000010 rate, 2.560975 seconds, 354720 images
Loaded: 0.000065 seconds

 11086: 0.009543, 0.013451 avg loss, 0.000010 rate, 2.565137 seconds, 354752 images
Loaded: 0.000042 seconds

 11087: 0.012420, 0.013348 avg loss, 0.000010 rate, 2.563875 seconds, 354784 images
Loaded: 0.000039 seconds

 11088: 0.024489, 0.014462 avg loss, 0.000010 rate, 2.562726 seconds, 354816 images
Loaded: 0.000069 seconds

 11089: 0.008692, 0.013885 avg loss, 0.000010 rate, 2.562060 seconds, 354848 images
Loaded: 0.000042 seconds

 11090: 0.012787, 0.013775 avg loss, 0.000010 rate, 2.562129 seconds, 354880 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000049 seconds

 11091: 0.010873, 0.013485 avg loss, 0.000010 rate, 3.023368 seconds, 354912 images
Loaded: 0.000051 seconds

 11092: 0.016100, 0.013746 avg loss, 0.000010 rate, 3.146452 seconds, 354944 images
Loaded: 0.000069 seconds

 11093: 0.007575, 0.013129 avg loss, 0.000010 rate, 3.140460 seconds, 354976 images
Loaded: 0.000045 seconds

 11094: 0.009571, 0.012773 avg loss, 0.000010 rate, 3.136499 seconds, 355008 images
Loaded: 0.000044 seconds

 11095: 0.017038, 0.013200 avg loss, 0.000010 rate, 3.144072 seconds, 355040 images
Loaded: 0.000043 seconds

 11096: 0.009302, 0.012810 avg loss, 0.000010 rate, 3.144112 seconds, 355072 images
Loaded: 0.000045 seconds

 11097: 0.010760, 0.012605 avg loss, 0.000010 rate, 3.136975 seconds, 355104 images
Loaded: 0.000052 seconds

 11098: 0.011688, 0.012513 avg loss, 0.000010 rate, 3.140926 seconds, 355136 images
Loaded: 0.000068 seconds

 11099: 0.008841, 0.012146 avg loss, 0.000010 rate, 3.140673 seconds, 355168 images
Loaded: 0.000066 seconds

 11100: 0.011712, 0.012103 avg loss, 0.000010 rate, 3.149174 seconds, 355200 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.262137 seconds

 11101: 0.022038, 0.013096 avg loss, 0.000010 rate, 2.460718 seconds, 355232 images
Loaded: 0.000046 seconds

 11102: 0.010020, 0.012789 avg loss, 0.000010 rate, 2.552868 seconds, 355264 images
Loaded: 0.000052 seconds

 11103: 0.010619, 0.012572 avg loss, 0.000010 rate, 2.552330 seconds, 355296 images
Loaded: 0.000049 seconds

 11104: 0.009739, 0.012288 avg loss, 0.000010 rate, 2.556665 seconds, 355328 images
Loaded: 0.000046 seconds

 11105: 0.006520, 0.011712 avg loss, 0.000010 rate, 2.556950 seconds, 355360 images
Loaded: 0.000043 seconds

 11106: 0.010486, 0.011589 avg loss, 0.000010 rate, 2.554441 seconds, 355392 images
Loaded: 0.000042 seconds

 11107: 0.013429, 0.011773 avg loss, 0.000010 rate, 2.559356 seconds, 355424 images
Loaded: 0.000077 seconds

 11108: 0.009922, 0.011588 avg loss, 0.000010 rate, 2.556498 seconds, 355456 images
Loaded: 0.000044 seconds

 11109: 0.008917, 0.011321 avg loss, 0.000010 rate, 2.559393 seconds, 355488 images
Loaded: 0.000045 seconds

 11110: 0.010158, 0.011205 avg loss, 0.000010 rate, 2.563808 seconds, 355520 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000061 seconds

 11111: 0.014348, 0.011519 avg loss, 0.000010 rate, 2.078054 seconds, 355552 images
Loaded: 0.000065 seconds

 11112: 0.012454, 0.011612 avg loss, 0.000010 rate, 2.167507 seconds, 355584 images
Loaded: 0.000064 seconds

 11113: 0.013444, 0.011796 avg loss, 0.000010 rate, 2.169116 seconds, 355616 images
Loaded: 0.000077 seconds

 11114: 0.011002, 0.011716 avg loss, 0.000010 rate, 2.168008 seconds, 355648 images
Loaded: 0.000056 seconds

 11115: 0.011351, 0.011680 avg loss, 0.000010 rate, 2.167103 seconds, 355680 images
Loaded: 0.000042 seconds

 11116: 0.015735, 0.012085 avg loss, 0.000010 rate, 2.172714 seconds, 355712 images
Loaded: 0.000048 seconds

 11117: 0.017811, 0.012658 avg loss, 0.000010 rate, 2.174350 seconds, 355744 images
Loaded: 0.000070 seconds

 11118: 0.011337, 0.012526 avg loss, 0.000010 rate, 2.168377 seconds, 355776 images
Loaded: 0.000061 seconds

 11119: 0.011331, 0.012406 avg loss, 0.000010 rate, 2.172457 seconds, 355808 images
Loaded: 0.000071 seconds

 11120: 0.012347, 0.012400 avg loss, 0.000010 rate, 2.166538 seconds, 355840 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.052216 seconds

 11121: 0.022188, 0.013379 avg loss, 0.000010 rate, 1.845514 seconds, 355872 images
Loaded: 0.000042 seconds

 11122: 0.009823, 0.013023 avg loss, 0.000010 rate, 1.921955 seconds, 355904 images
Loaded: 0.000075 seconds

 11123: 0.013441, 0.013065 avg loss, 0.000010 rate, 1.924437 seconds, 355936 images
Loaded: 0.000081 seconds

 11124: 0.028261, 0.014585 avg loss, 0.000010 rate, 1.935552 seconds, 355968 images
Loaded: 0.000048 seconds

 11125: 0.015902, 0.014717 avg loss, 0.000010 rate, 1.926938 seconds, 356000 images
Loaded: 0.000040 seconds

 11126: 0.011004, 0.014345 avg loss, 0.000010 rate, 1.919928 seconds, 356032 images
Loaded: 0.000051 seconds

 11127: 0.041117, 0.017022 avg loss, 0.000010 rate, 1.930737 seconds, 356064 images
Loaded: 0.000043 seconds

 11128: 0.018056, 0.017126 avg loss, 0.000010 rate, 1.933878 seconds, 356096 images
Loaded: 0.000070 seconds

 11129: 0.015197, 0.016933 avg loss, 0.000010 rate, 1.926139 seconds, 356128 images
Loaded: 0.000043 seconds

 11130: 0.040186, 0.019258 avg loss, 0.000010 rate, 1.927854 seconds, 356160 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000037 seconds

 11131: 0.015864, 0.018919 avg loss, 0.000010 rate, 4.514512 seconds, 356192 images
Loaded: 0.000065 seconds

 11132: 0.024963, 0.019523 avg loss, 0.000010 rate, 4.667873 seconds, 356224 images
Loaded: 0.000071 seconds

 11133: 0.018545, 0.019425 avg loss, 0.000010 rate, 4.667462 seconds, 356256 images
Loaded: 0.000071 seconds

 11134: 0.011172, 0.018600 avg loss, 0.000010 rate, 4.677641 seconds, 356288 images
Loaded: 0.000045 seconds

 11135: 0.017400, 0.018480 avg loss, 0.000010 rate, 4.664513 seconds, 356320 images
Loaded: 0.000037 seconds

 11136: 0.011385, 0.017771 avg loss, 0.000010 rate, 4.680854 seconds, 356352 images
Loaded: 0.000075 seconds

 11137: 0.020362, 0.018030 avg loss, 0.000010 rate, 4.663601 seconds, 356384 images
Loaded: 0.000046 seconds

 11138: 0.053054, 0.021532 avg loss, 0.000010 rate, 4.685882 seconds, 356416 images
Loaded: 0.000086 seconds

 11139: 0.018140, 0.021193 avg loss, 0.000010 rate, 4.677005 seconds, 356448 images
Loaded: 0.000047 seconds

 11140: 0.012717, 0.020345 avg loss, 0.000010 rate, 4.674874 seconds, 356480 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000036 seconds

 11141: 0.007883, 0.019099 avg loss, 0.000010 rate, 3.279115 seconds, 356512 images
Loaded: 0.000045 seconds

 11142: 0.012311, 0.018420 avg loss, 0.000010 rate, 3.393686 seconds, 356544 images
Loaded: 0.000040 seconds

 11143: 0.015818, 0.018160 avg loss, 0.000010 rate, 3.385760 seconds, 356576 images
Loaded: 0.000042 seconds

 11144: 0.011906, 0.017535 avg loss, 0.000010 rate, 3.382895 seconds, 356608 images
Loaded: 0.000046 seconds

 11145: 0.010697, 0.016851 avg loss, 0.000010 rate, 3.383303 seconds, 356640 images
Loaded: 0.000081 seconds

 11146: 0.010066, 0.016172 avg loss, 0.000010 rate, 3.395508 seconds, 356672 images
Loaded: 0.000070 seconds

 11147: 0.007000, 0.015255 avg loss, 0.000010 rate, 3.382980 seconds, 356704 images
Loaded: 0.000047 seconds

 11148: 0.017463, 0.015476 avg loss, 0.000010 rate, 3.392327 seconds, 356736 images
Loaded: 0.000043 seconds

 11149: 0.014104, 0.015339 avg loss, 0.000010 rate, 3.379871 seconds, 356768 images
Loaded: 0.000044 seconds

 11150: 0.014729, 0.015278 avg loss, 0.000010 rate, 3.397913 seconds, 356800 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11151: 0.015358, 0.015286 avg loss, 0.000010 rate, 4.505151 seconds, 356832 images
Loaded: 0.000043 seconds

 11152: 0.012239, 0.014981 avg loss, 0.000010 rate, 4.666235 seconds, 356864 images
Loaded: 0.000070 seconds

 11153: 0.010712, 0.014554 avg loss, 0.000010 rate, 4.665219 seconds, 356896 images
Loaded: 0.000044 seconds

 11154: 0.024093, 0.015508 avg loss, 0.000010 rate, 4.674530 seconds, 356928 images
Loaded: 0.000041 seconds

 11155: 0.012859, 0.015243 avg loss, 0.000010 rate, 4.665868 seconds, 356960 images
Loaded: 0.000067 seconds

 11156: 0.008679, 0.014587 avg loss, 0.000010 rate, 4.683501 seconds, 356992 images
Loaded: 0.000068 seconds

 11157: 0.031853, 0.016313 avg loss, 0.000010 rate, 4.677017 seconds, 357024 images
Loaded: 0.000078 seconds

 11158: 0.010649, 0.015747 avg loss, 0.000010 rate, 4.676069 seconds, 357056 images
Loaded: 0.000055 seconds

 11159: 0.014773, 0.015650 avg loss, 0.000010 rate, 4.669372 seconds, 357088 images
Loaded: 0.000046 seconds

 11160: 0.015418, 0.015627 avg loss, 0.000010 rate, 4.675385 seconds, 357120 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000060 seconds

 11161: 0.012523, 0.015316 avg loss, 0.000010 rate, 3.270670 seconds, 357152 images
Loaded: 0.000036 seconds

 11162: 0.009903, 0.014775 avg loss, 0.000010 rate, 3.386726 seconds, 357184 images
Loaded: 0.000068 seconds

 11163: 0.007778, 0.014075 avg loss, 0.000010 rate, 3.398808 seconds, 357216 images
Loaded: 0.000062 seconds

 11164: 0.012100, 0.013878 avg loss, 0.000010 rate, 3.394958 seconds, 357248 images
Loaded: 0.000069 seconds

 11165: 0.011606, 0.013651 avg loss, 0.000010 rate, 3.394157 seconds, 357280 images
Loaded: 0.000047 seconds

 11166: 0.013073, 0.013593 avg loss, 0.000010 rate, 3.394147 seconds, 357312 images
Loaded: 0.000071 seconds

 11167: 0.016019, 0.013835 avg loss, 0.000010 rate, 3.406404 seconds, 357344 images
Loaded: 0.000043 seconds

 11168: 0.015720, 0.014024 avg loss, 0.000010 rate, 3.388765 seconds, 357376 images
Loaded: 0.000043 seconds

 11169: 0.010730, 0.013694 avg loss, 0.000010 rate, 3.395854 seconds, 357408 images
Loaded: 0.000043 seconds

 11170: 0.007655, 0.013091 avg loss, 0.000010 rate, 3.394125 seconds, 357440 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.082537 seconds

 11171: 0.009878, 0.012769 avg loss, 0.000010 rate, 3.027544 seconds, 357472 images
Loaded: 0.000041 seconds

 11172: 0.010989, 0.012591 avg loss, 0.000010 rate, 3.143323 seconds, 357504 images
Loaded: 0.000043 seconds

 11173: 0.015573, 0.012889 avg loss, 0.000010 rate, 3.150980 seconds, 357536 images
Loaded: 0.000044 seconds

 11174: 0.011657, 0.012766 avg loss, 0.000010 rate, 3.143335 seconds, 357568 images
Loaded: 0.000035 seconds

 11175: 0.011937, 0.012683 avg loss, 0.000010 rate, 3.150418 seconds, 357600 images
Loaded: 0.000067 seconds

 11176: 0.012110, 0.012626 avg loss, 0.000010 rate, 3.149870 seconds, 357632 images
Loaded: 0.000060 seconds

 11177: 0.010137, 0.012377 avg loss, 0.000010 rate, 3.141509 seconds, 357664 images
Loaded: 0.000044 seconds

 11178: 0.008833, 0.012023 avg loss, 0.000010 rate, 3.150524 seconds, 357696 images
Loaded: 0.000079 seconds

 11179: 0.009194, 0.011740 avg loss, 0.000010 rate, 3.148724 seconds, 357728 images
Loaded: 0.000042 seconds

 11180: 0.009769, 0.011543 avg loss, 0.000010 rate, 3.146113 seconds, 357760 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 11181: 0.007534, 0.011142 avg loss, 0.000010 rate, 2.082155 seconds, 357792 images
Loaded: 0.000044 seconds

 11182: 0.012929, 0.011321 avg loss, 0.000010 rate, 2.164584 seconds, 357824 images
Loaded: 0.000052 seconds

 11183: 0.013582, 0.011547 avg loss, 0.000010 rate, 2.171437 seconds, 357856 images
Loaded: 0.000045 seconds

 11184: 0.011566, 0.011549 avg loss, 0.000010 rate, 2.174338 seconds, 357888 images
Loaded: 0.000038 seconds

 11185: 0.012640, 0.011658 avg loss, 0.000010 rate, 2.173157 seconds, 357920 images
Loaded: 0.000046 seconds

 11186: 0.011068, 0.011599 avg loss, 0.000010 rate, 2.170028 seconds, 357952 images
Loaded: 0.000068 seconds

 11187: 0.013226, 0.011762 avg loss, 0.000010 rate, 2.170608 seconds, 357984 images
Loaded: 0.000042 seconds

 11188: 0.014485, 0.012034 avg loss, 0.000010 rate, 2.175094 seconds, 358016 images
Loaded: 0.000048 seconds

 11189: 0.015276, 0.012358 avg loss, 0.000010 rate, 2.171184 seconds, 358048 images
Loaded: 0.000041 seconds

 11190: 0.012621, 0.012384 avg loss, 0.000010 rate, 2.177350 seconds, 358080 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11191: 0.014448, 0.012591 avg loss, 0.000010 rate, 4.509782 seconds, 358112 images
Loaded: 0.000035 seconds

 11192: 0.017160, 0.013048 avg loss, 0.000010 rate, 4.679108 seconds, 358144 images
Loaded: 0.000078 seconds

 11193: 0.018480, 0.013591 avg loss, 0.000010 rate, 4.674916 seconds, 358176 images
Loaded: 0.000045 seconds

 11194: 0.010949, 0.013327 avg loss, 0.000010 rate, 4.677856 seconds, 358208 images
Loaded: 0.000063 seconds

 11195: 0.014632, 0.013457 avg loss, 0.000010 rate, 4.670576 seconds, 358240 images
Loaded: 0.000046 seconds

 11196: 0.017168, 0.013828 avg loss, 0.000010 rate, 4.684700 seconds, 358272 images
Loaded: 0.000069 seconds

 11197: 0.010819, 0.013527 avg loss, 0.000010 rate, 4.668994 seconds, 358304 images
Loaded: 0.000093 seconds

 11198: 0.014694, 0.013644 avg loss, 0.000010 rate, 4.679478 seconds, 358336 images
Loaded: 0.000056 seconds

 11199: 0.012212, 0.013501 avg loss, 0.000010 rate, 4.685443 seconds, 358368 images
Loaded: 0.000053 seconds

 11200: 0.014709, 0.013622 avg loss, 0.000010 rate, 4.680100 seconds, 358400 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.221054 seconds

 11201: 0.012756, 0.013535 avg loss, 0.000010 rate, 3.278642 seconds, 358432 images
Loaded: 0.000057 seconds

 11202: 0.012856, 0.013467 avg loss, 0.000010 rate, 3.393486 seconds, 358464 images
Loaded: 0.000067 seconds

 11203: 0.015640, 0.013684 avg loss, 0.000010 rate, 3.385199 seconds, 358496 images
Loaded: 0.000047 seconds

 11204: 0.009754, 0.013291 avg loss, 0.000010 rate, 3.397001 seconds, 358528 images
Loaded: 0.000052 seconds

 11205: 0.011638, 0.013126 avg loss, 0.000010 rate, 3.388852 seconds, 358560 images
Loaded: 0.000053 seconds

 11206: 0.009659, 0.012779 avg loss, 0.000010 rate, 3.398145 seconds, 358592 images
Loaded: 0.000076 seconds

 11207: 0.008849, 0.012386 avg loss, 0.000010 rate, 3.388485 seconds, 358624 images
Loaded: 0.000041 seconds

 11208: 0.014016, 0.012549 avg loss, 0.000010 rate, 3.395648 seconds, 358656 images
Loaded: 0.000051 seconds

 11209: 0.015627, 0.012857 avg loss, 0.000010 rate, 3.381771 seconds, 358688 images
Loaded: 0.000050 seconds

 11210: 0.005958, 0.012167 avg loss, 0.000010 rate, 3.386188 seconds, 358720 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11211: 0.010395, 0.011990 avg loss, 0.000010 rate, 4.171227 seconds, 358752 images
Loaded: 0.000042 seconds

 11212: 0.010971, 0.011888 avg loss, 0.000010 rate, 4.327903 seconds, 358784 images
Loaded: 0.000043 seconds

 11213: 0.013887, 0.012088 avg loss, 0.000010 rate, 4.317432 seconds, 358816 images
Loaded: 0.000073 seconds

 11214: 0.006822, 0.011561 avg loss, 0.000010 rate, 4.328365 seconds, 358848 images
Loaded: 0.000079 seconds

 11215: 0.014025, 0.011808 avg loss, 0.000010 rate, 4.320915 seconds, 358880 images
Loaded: 0.000044 seconds

 11216: 0.014519, 0.012079 avg loss, 0.000010 rate, 4.321367 seconds, 358912 images
Loaded: 0.000048 seconds

 11217: 0.009360, 0.011807 avg loss, 0.000010 rate, 4.328187 seconds, 358944 images
Loaded: 0.000040 seconds

 11218: 0.008459, 0.011472 avg loss, 0.000010 rate, 4.318040 seconds, 358976 images
Loaded: 0.000046 seconds

 11219: 0.015205, 0.011845 avg loss, 0.000010 rate, 4.315395 seconds, 359008 images
Loaded: 0.000042 seconds

 11220: 0.008856, 0.011546 avg loss, 0.000010 rate, 4.335720 seconds, 359040 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.024364 seconds

 11221: 0.010956, 0.011487 avg loss, 0.000010 rate, 2.700471 seconds, 359072 images
Loaded: 0.000044 seconds

 11222: 0.009801, 0.011319 avg loss, 0.000010 rate, 2.799811 seconds, 359104 images
Loaded: 0.000044 seconds

 11223: 0.010191, 0.011206 avg loss, 0.000010 rate, 2.798591 seconds, 359136 images
Loaded: 0.000043 seconds

 11224: 0.010889, 0.011174 avg loss, 0.000010 rate, 2.798125 seconds, 359168 images
Loaded: 0.000067 seconds

 11225: 0.009950, 0.011052 avg loss, 0.000010 rate, 2.799810 seconds, 359200 images
Loaded: 0.000044 seconds

 11226: 0.015552, 0.011502 avg loss, 0.000010 rate, 2.801070 seconds, 359232 images
Loaded: 0.000044 seconds

 11227: 0.014171, 0.011769 avg loss, 0.000010 rate, 2.803834 seconds, 359264 images
Loaded: 0.000073 seconds

 11228: 0.011115, 0.011703 avg loss, 0.000010 rate, 2.800427 seconds, 359296 images
Loaded: 0.000043 seconds

 11229: 0.009712, 0.011504 avg loss, 0.000010 rate, 2.799171 seconds, 359328 images
Loaded: 0.000042 seconds

 11230: 0.010795, 0.011433 avg loss, 0.000010 rate, 2.807126 seconds, 359360 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000087 seconds

 11231: 0.011412, 0.011431 avg loss, 0.000010 rate, 3.021784 seconds, 359392 images
Loaded: 0.000047 seconds

 11232: 0.008755, 0.011164 avg loss, 0.000010 rate, 3.129152 seconds, 359424 images
Loaded: 0.000037 seconds

 11233: 0.011044, 0.011152 avg loss, 0.000010 rate, 3.135852 seconds, 359456 images
Loaded: 0.000043 seconds

 11234: 0.013128, 0.011349 avg loss, 0.000010 rate, 3.130195 seconds, 359488 images
Loaded: 0.000047 seconds

 11235: 0.008597, 0.011074 avg loss, 0.000010 rate, 3.133634 seconds, 359520 images
Loaded: 0.000120 seconds

 11236: 0.010418, 0.011008 avg loss, 0.000010 rate, 3.136577 seconds, 359552 images
Loaded: 0.000051 seconds

 11237: 0.009829, 0.010890 avg loss, 0.000010 rate, 3.128060 seconds, 359584 images
Loaded: 0.000046 seconds

 11238: 0.013875, 0.011189 avg loss, 0.000010 rate, 3.135941 seconds, 359616 images
Loaded: 0.000040 seconds

 11239: 0.016595, 0.011730 avg loss, 0.000010 rate, 3.120683 seconds, 359648 images
Loaded: 0.000051 seconds

 11240: 0.009194, 0.011476 avg loss, 0.000010 rate, 3.131824 seconds, 359680 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000055 seconds

 11241: 0.014729, 0.011801 avg loss, 0.000010 rate, 4.486456 seconds, 359712 images
Loaded: 0.000044 seconds

 11242: 0.011094, 0.011731 avg loss, 0.000010 rate, 4.652304 seconds, 359744 images
Loaded: 0.000038 seconds

 11243: 0.023923, 0.012950 avg loss, 0.000010 rate, 4.644635 seconds, 359776 images
Loaded: 0.000044 seconds

 11244: 0.012391, 0.012894 avg loss, 0.000010 rate, 4.655342 seconds, 359808 images
Loaded: 0.000043 seconds

 11245: 0.012261, 0.012831 avg loss, 0.000010 rate, 4.646138 seconds, 359840 images
Loaded: 0.000051 seconds

 11246: 0.013433, 0.012891 avg loss, 0.000010 rate, 4.654997 seconds, 359872 images
Loaded: 0.000049 seconds

 11247: 0.012873, 0.012889 avg loss, 0.000010 rate, 4.653646 seconds, 359904 images
Loaded: 0.000034 seconds

 11248: 0.011061, 0.012706 avg loss, 0.000010 rate, 4.660305 seconds, 359936 images
Loaded: 0.000062 seconds

 11249: 0.007514, 0.012187 avg loss, 0.000010 rate, 4.667981 seconds, 359968 images
Loaded: 0.000101 seconds

 11250: 0.009764, 0.011945 avg loss, 0.000010 rate, 4.682915 seconds, 360000 images
Resizing
320 x 320 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000045 seconds

 11251: 0.028851, 0.013635 avg loss, 0.000010 rate, 1.630820 seconds, 360032 images
Loaded: 0.000039 seconds

 11252: 0.017482, 0.014020 avg loss, 0.000010 rate, 1.692631 seconds, 360064 images
Loaded: 0.000070 seconds

 11253: 0.015903, 0.014208 avg loss, 0.000010 rate, 1.709169 seconds, 360096 images
Loaded: 0.000041 seconds

 11254: 0.017461, 0.014534 avg loss, 0.000010 rate, 1.692645 seconds, 360128 images
Loaded: 0.000042 seconds

 11255: 0.022438, 0.015324 avg loss, 0.000010 rate, 1.689655 seconds, 360160 images
Loaded: 0.000073 seconds

 11256: 0.075087, 0.021300 avg loss, 0.000010 rate, 1.694449 seconds, 360192 images
Loaded: 0.000054 seconds

 11257: 0.024909, 0.021661 avg loss, 0.000010 rate, 1.697225 seconds, 360224 images
Loaded: 0.000042 seconds

 11258: 0.033066, 0.022802 avg loss, 0.000010 rate, 1.694023 seconds, 360256 images
Loaded: 0.000099 seconds

 11259: 0.028634, 0.023385 avg loss, 0.000010 rate, 1.696560 seconds, 360288 images
Loaded: 0.000043 seconds

 11260: 0.035446, 0.024591 avg loss, 0.000010 rate, 1.693312 seconds, 360320 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000038 seconds

 11261: 0.016042, 0.023736 avg loss, 0.000010 rate, 1.851203 seconds, 360352 images
Loaded: 0.000037 seconds

 11262: 0.013121, 0.022675 avg loss, 0.000010 rate, 1.923027 seconds, 360384 images
Loaded: 0.000068 seconds

 11263: 0.014116, 0.021819 avg loss, 0.000010 rate, 1.930952 seconds, 360416 images
Loaded: 0.000039 seconds

 11264: 0.014732, 0.021110 avg loss, 0.000010 rate, 1.924948 seconds, 360448 images
Loaded: 0.000044 seconds

 11265: 0.016195, 0.020619 avg loss, 0.000010 rate, 1.927172 seconds, 360480 images
Loaded: 0.000067 seconds

 11266: 0.013254, 0.019882 avg loss, 0.000010 rate, 1.930890 seconds, 360512 images
Loaded: 0.000046 seconds

 11267: 0.012844, 0.019178 avg loss, 0.000010 rate, 1.922287 seconds, 360544 images
Loaded: 0.000068 seconds

 11268: 0.015595, 0.018820 avg loss, 0.000010 rate, 1.926766 seconds, 360576 images
Loaded: 0.000067 seconds

 11269: 0.014514, 0.018389 avg loss, 0.000010 rate, 1.929158 seconds, 360608 images
Loaded: 0.000040 seconds

 11270: 0.013582, 0.017909 avg loss, 0.000010 rate, 1.925798 seconds, 360640 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.097085 seconds

 11271: 0.015429, 0.017661 avg loss, 0.000010 rate, 1.844449 seconds, 360672 images
Loaded: 0.000065 seconds

 11272: 0.016761, 0.017571 avg loss, 0.000010 rate, 1.918849 seconds, 360704 images
Loaded: 0.000043 seconds

 11273: 0.010688, 0.016882 avg loss, 0.000010 rate, 1.925817 seconds, 360736 images
Loaded: 0.000039 seconds

 11274: 0.026493, 0.017843 avg loss, 0.000010 rate, 1.921267 seconds, 360768 images
Loaded: 0.000042 seconds

 11275: 0.017746, 0.017834 avg loss, 0.000010 rate, 1.925777 seconds, 360800 images
Loaded: 0.000055 seconds

 11276: 0.029347, 0.018985 avg loss, 0.000010 rate, 1.927651 seconds, 360832 images
Loaded: 0.000043 seconds

 11277: 0.024080, 0.019494 avg loss, 0.000010 rate, 1.927129 seconds, 360864 images
Loaded: 0.000057 seconds

 11278: 0.014786, 0.019024 avg loss, 0.000010 rate, 1.927156 seconds, 360896 images
Loaded: 0.000043 seconds

 11279: 0.015250, 0.018646 avg loss, 0.000010 rate, 1.932624 seconds, 360928 images
Loaded: 0.000069 seconds

 11280: 0.015853, 0.018367 avg loss, 0.000010 rate, 1.920358 seconds, 360960 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000063 seconds

 11281: 0.012458, 0.017776 avg loss, 0.000010 rate, 2.455624 seconds, 360992 images
Loaded: 0.000039 seconds

 11282: 0.012488, 0.017247 avg loss, 0.000010 rate, 2.555839 seconds, 361024 images
Loaded: 0.000048 seconds

 11283: 0.009781, 0.016501 avg loss, 0.000010 rate, 2.557885 seconds, 361056 images
Loaded: 0.000044 seconds

 11284: 0.007193, 0.015570 avg loss, 0.000010 rate, 2.556965 seconds, 361088 images
Loaded: 0.000069 seconds

 11285: 0.031866, 0.017199 avg loss, 0.000010 rate, 2.568337 seconds, 361120 images
Loaded: 0.000071 seconds

 11286: 0.014462, 0.016926 avg loss, 0.000010 rate, 2.559448 seconds, 361152 images
Loaded: 0.000070 seconds

 11287: 0.012603, 0.016493 avg loss, 0.000010 rate, 2.559291 seconds, 361184 images
Loaded: 0.000067 seconds

 11288: 0.012140, 0.016058 avg loss, 0.000010 rate, 2.561542 seconds, 361216 images
Loaded: 0.000072 seconds

 11289: 0.012050, 0.015657 avg loss, 0.000010 rate, 2.568808 seconds, 361248 images
Loaded: 0.000042 seconds

 11290: 0.020277, 0.016119 avg loss, 0.000010 rate, 2.571265 seconds, 361280 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.144914 seconds

 11291: 0.014002, 0.015907 avg loss, 0.000010 rate, 1.839982 seconds, 361312 images
Loaded: 0.000038 seconds

 11292: 0.018684, 0.016185 avg loss, 0.000010 rate, 1.920213 seconds, 361344 images
Loaded: 0.000073 seconds

 11293: 0.019443, 0.016511 avg loss, 0.000010 rate, 1.922205 seconds, 361376 images
Loaded: 0.000044 seconds

 11294: 0.012964, 0.016156 avg loss, 0.000010 rate, 1.923038 seconds, 361408 images
Loaded: 0.000042 seconds

 11295: 0.012972, 0.015838 avg loss, 0.000010 rate, 1.925811 seconds, 361440 images
Loaded: 0.000071 seconds

 11296: 0.021448, 0.016399 avg loss, 0.000010 rate, 1.929408 seconds, 361472 images
Loaded: 0.000046 seconds

 11297: 0.012142, 0.015973 avg loss, 0.000010 rate, 1.926677 seconds, 361504 images
Loaded: 0.000074 seconds

 11298: 0.017762, 0.016152 avg loss, 0.000010 rate, 1.920053 seconds, 361536 images
Loaded: 0.000085 seconds

 11299: 0.023356, 0.016872 avg loss, 0.000010 rate, 1.923666 seconds, 361568 images
Loaded: 0.000066 seconds

 11300: 0.013983, 0.016583 avg loss, 0.000010 rate, 1.931058 seconds, 361600 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11301: 0.010899, 0.016015 avg loss, 0.000010 rate, 2.087504 seconds, 361632 images
Loaded: 0.000068 seconds

 11302: 0.012785, 0.015692 avg loss, 0.000010 rate, 2.168572 seconds, 361664 images
Loaded: 0.000040 seconds

 11303: 0.019662, 0.016089 avg loss, 0.000010 rate, 2.162808 seconds, 361696 images
Loaded: 0.000043 seconds

 11304: 0.009233, 0.015403 avg loss, 0.000010 rate, 2.170693 seconds, 361728 images
Loaded: 0.000047 seconds

 11305: 0.014267, 0.015290 avg loss, 0.000010 rate, 2.168315 seconds, 361760 images
Loaded: 0.000081 seconds

 11306: 0.008327, 0.014593 avg loss, 0.000010 rate, 2.175031 seconds, 361792 images
Loaded: 0.000072 seconds

 11307: 0.011293, 0.014263 avg loss, 0.000010 rate, 2.175630 seconds, 361824 images
Loaded: 0.000071 seconds

 11308: 0.010999, 0.013937 avg loss, 0.000010 rate, 2.167302 seconds, 361856 images
Loaded: 0.000068 seconds

 11309: 0.013090, 0.013852 avg loss, 0.000010 rate, 2.172395 seconds, 361888 images
Loaded: 0.000064 seconds

 11310: 0.008032, 0.013270 avg loss, 0.000010 rate, 2.166289 seconds, 361920 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11311: 0.013861, 0.013329 avg loss, 0.000010 rate, 4.174922 seconds, 361952 images
Loaded: 0.000042 seconds

 11312: 0.014287, 0.013425 avg loss, 0.000010 rate, 4.329119 seconds, 361984 images
Loaded: 0.000077 seconds

 11313: 0.008793, 0.012962 avg loss, 0.000010 rate, 4.319590 seconds, 362016 images
Loaded: 0.000045 seconds

 11314: 0.012276, 0.012893 avg loss, 0.000010 rate, 4.332413 seconds, 362048 images
Loaded: 0.000075 seconds

 11315: 0.021435, 0.013747 avg loss, 0.000010 rate, 4.317928 seconds, 362080 images
Loaded: 0.000046 seconds

 11316: 0.007094, 0.013082 avg loss, 0.000010 rate, 4.336953 seconds, 362112 images
Loaded: 0.000046 seconds

 11317: 0.009304, 0.012704 avg loss, 0.000010 rate, 4.335478 seconds, 362144 images
Loaded: 0.000045 seconds

 11318: 0.009213, 0.012355 avg loss, 0.000010 rate, 4.343013 seconds, 362176 images
Loaded: 0.000044 seconds

 11319: 0.009546, 0.012074 avg loss, 0.000010 rate, 4.339128 seconds, 362208 images
Loaded: 0.000045 seconds

 11320: 0.028920, 0.013759 avg loss, 0.000010 rate, 4.351517 seconds, 362240 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000050 seconds

 11321: 0.018570, 0.014240 avg loss, 0.000010 rate, 1.856203 seconds, 362272 images
Loaded: 0.000042 seconds

 11322: 0.009771, 0.013793 avg loss, 0.000010 rate, 1.927381 seconds, 362304 images
Loaded: 0.000076 seconds

 11323: 0.011587, 0.013572 avg loss, 0.000010 rate, 1.925278 seconds, 362336 images
Loaded: 0.000043 seconds

 11324: 0.011999, 0.013415 avg loss, 0.000010 rate, 1.923356 seconds, 362368 images
Loaded: 0.000043 seconds

 11325: 0.026492, 0.014723 avg loss, 0.000010 rate, 1.923968 seconds, 362400 images
Loaded: 0.000044 seconds

 11326: 0.020234, 0.015274 avg loss, 0.000010 rate, 1.926697 seconds, 362432 images
Loaded: 0.000078 seconds

 11327: 0.022212, 0.015968 avg loss, 0.000010 rate, 1.921544 seconds, 362464 images
Loaded: 0.000066 seconds

 11328: 0.017449, 0.016116 avg loss, 0.000010 rate, 1.919653 seconds, 362496 images
Loaded: 0.000031 seconds

 11329: 0.017711, 0.016275 avg loss, 0.000010 rate, 1.913008 seconds, 362528 images
Loaded: 0.000039 seconds

 11330: 0.047810, 0.019429 avg loss, 0.000010 rate, 1.912009 seconds, 362560 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.117347 seconds

 11331: 0.010257, 0.018512 avg loss, 0.000010 rate, 1.842733 seconds, 362592 images
Loaded: 0.000031 seconds

 11332: 0.040312, 0.020692 avg loss, 0.000010 rate, 1.918048 seconds, 362624 images
Loaded: 0.000043 seconds

 11333: 0.011915, 0.019814 avg loss, 0.000010 rate, 1.928843 seconds, 362656 images
Loaded: 0.000059 seconds

 11334: 0.015914, 0.019424 avg loss, 0.000010 rate, 1.924568 seconds, 362688 images
Loaded: 0.000042 seconds

 11335: 0.016812, 0.019163 avg loss, 0.000010 rate, 1.917629 seconds, 362720 images
Loaded: 0.000040 seconds

 11336: 0.017309, 0.018977 avg loss, 0.000010 rate, 1.919915 seconds, 362752 images
Loaded: 0.000089 seconds

 11337: 0.010749, 0.018155 avg loss, 0.000010 rate, 1.924512 seconds, 362784 images
Loaded: 0.000070 seconds

 11338: 0.023214, 0.018661 avg loss, 0.000010 rate, 1.918226 seconds, 362816 images
Loaded: 0.000044 seconds

 11339: 0.015765, 0.018371 avg loss, 0.000010 rate, 1.925891 seconds, 362848 images
Loaded: 0.000065 seconds

 11340: 0.016653, 0.018199 avg loss, 0.000010 rate, 1.914558 seconds, 362880 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11341: 0.010177, 0.017397 avg loss, 0.000010 rate, 3.257753 seconds, 362912 images
Loaded: 0.000042 seconds

 11342: 0.013021, 0.016959 avg loss, 0.000010 rate, 3.373334 seconds, 362944 images
Loaded: 0.000042 seconds

 11343: 0.009264, 0.016190 avg loss, 0.000010 rate, 3.360267 seconds, 362976 images
Loaded: 0.000043 seconds

 11344: 0.021822, 0.016753 avg loss, 0.000010 rate, 3.380831 seconds, 363008 images
Loaded: 0.000073 seconds

 11345: 0.009914, 0.016069 avg loss, 0.000010 rate, 3.362783 seconds, 363040 images
Loaded: 0.000044 seconds

 11346: 0.009386, 0.015401 avg loss, 0.000010 rate, 3.376624 seconds, 363072 images
Loaded: 0.000053 seconds

 11347: 0.013351, 0.015196 avg loss, 0.000010 rate, 3.362693 seconds, 363104 images
Loaded: 0.000052 seconds

 11348: 0.011899, 0.014866 avg loss, 0.000010 rate, 3.379539 seconds, 363136 images
Loaded: 0.000074 seconds

 11349: 0.007976, 0.014177 avg loss, 0.000010 rate, 3.362338 seconds, 363168 images
Loaded: 0.000043 seconds

 11350: 0.016986, 0.014458 avg loss, 0.000010 rate, 3.381735 seconds, 363200 images
Resizing
320 x 320 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11351: 0.027484, 0.015761 avg loss, 0.000010 rate, 1.623455 seconds, 363232 images
Loaded: 0.000038 seconds

 11352: 0.033162, 0.017501 avg loss, 0.000010 rate, 1.690874 seconds, 363264 images
Loaded: 0.000044 seconds

 11353: 0.027646, 0.018515 avg loss, 0.000010 rate, 1.689776 seconds, 363296 images
Loaded: 0.000085 seconds

 11354: 0.024610, 0.019125 avg loss, 0.000010 rate, 1.693980 seconds, 363328 images
Loaded: 0.000033 seconds

 11355: 0.018423, 0.019055 avg loss, 0.000010 rate, 1.683978 seconds, 363360 images
Loaded: 0.000042 seconds

 11356: 0.039194, 0.021069 avg loss, 0.000010 rate, 1.688236 seconds, 363392 images
Loaded: 0.000072 seconds

 11357: 0.076946, 0.026656 avg loss, 0.000010 rate, 1.689895 seconds, 363424 images
Loaded: 0.000032 seconds

 11358: 0.036526, 0.027643 avg loss, 0.000010 rate, 1.689890 seconds, 363456 images
Loaded: 0.000048 seconds

 11359: 0.019955, 0.026874 avg loss, 0.000010 rate, 1.692067 seconds, 363488 images
Loaded: 0.000052 seconds

 11360: 0.035768, 0.027764 avg loss, 0.000010 rate, 1.694496 seconds, 363520 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11361: 0.023021, 0.027290 avg loss, 0.000010 rate, 1.842155 seconds, 363552 images
Loaded: 0.000041 seconds

 11362: 0.015882, 0.026149 avg loss, 0.000010 rate, 1.920053 seconds, 363584 images
Loaded: 0.000052 seconds

 11363: 0.018940, 0.025428 avg loss, 0.000010 rate, 1.921770 seconds, 363616 images
Loaded: 0.000041 seconds

 11364: 0.023511, 0.025236 avg loss, 0.000010 rate, 1.918295 seconds, 363648 images
Loaded: 0.000042 seconds

 11365: 0.011191, 0.023832 avg loss, 0.000010 rate, 1.913917 seconds, 363680 images
Loaded: 0.000045 seconds

 11366: 0.018555, 0.023304 avg loss, 0.000010 rate, 1.927714 seconds, 363712 images
Loaded: 0.000041 seconds

 11367: 0.022495, 0.023223 avg loss, 0.000010 rate, 1.926776 seconds, 363744 images
Loaded: 0.000068 seconds

 11368: 0.013202, 0.022221 avg loss, 0.000010 rate, 1.921721 seconds, 363776 images
Loaded: 0.000064 seconds

 11369: 0.013028, 0.021302 avg loss, 0.000010 rate, 1.920640 seconds, 363808 images
Loaded: 0.000040 seconds

 11370: 0.018389, 0.021011 avg loss, 0.000010 rate, 1.918772 seconds, 363840 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11371: 0.012280, 0.020138 avg loss, 0.000010 rate, 4.160495 seconds, 363872 images
Loaded: 0.000047 seconds

 11372: 0.023345, 0.020458 avg loss, 0.000010 rate, 4.326855 seconds, 363904 images
Loaded: 0.000070 seconds

 11373: 0.007610, 0.019173 avg loss, 0.000010 rate, 4.329316 seconds, 363936 images
Loaded: 0.000067 seconds

 11374: 0.009681, 0.018224 avg loss, 0.000010 rate, 4.324921 seconds, 363968 images
Loaded: 0.000053 seconds

 11375: 0.007515, 0.017153 avg loss, 0.000010 rate, 4.327738 seconds, 364000 images
Loaded: 0.000043 seconds

 11376: 0.012381, 0.016676 avg loss, 0.000010 rate, 4.340684 seconds, 364032 images
Loaded: 0.000070 seconds

 11377: 0.011272, 0.016136 avg loss, 0.000010 rate, 4.334293 seconds, 364064 images
Loaded: 0.000042 seconds

 11378: 0.009849, 0.015507 avg loss, 0.000010 rate, 4.346316 seconds, 364096 images
Loaded: 0.000078 seconds

 11379: 0.011897, 0.015146 avg loss, 0.000010 rate, 4.352718 seconds, 364128 images
Loaded: 0.000049 seconds

 11380: 0.011337, 0.014765 avg loss, 0.000010 rate, 4.346230 seconds, 364160 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000037 seconds

 11381: 0.012422, 0.014531 avg loss, 0.000010 rate, 4.511403 seconds, 364192 images
Loaded: 0.000070 seconds

 11382: 0.012503, 0.014328 avg loss, 0.000010 rate, 4.660543 seconds, 364224 images
Loaded: 0.000044 seconds

 11383: 0.012834, 0.014179 avg loss, 0.000010 rate, 4.669217 seconds, 364256 images
Loaded: 0.000068 seconds

 11384: 0.011817, 0.013942 avg loss, 0.000010 rate, 4.676394 seconds, 364288 images
Loaded: 0.000048 seconds

 11385: 0.021018, 0.014650 avg loss, 0.000010 rate, 4.679510 seconds, 364320 images
Loaded: 0.000044 seconds

 11386: 0.019041, 0.015089 avg loss, 0.000010 rate, 4.683667 seconds, 364352 images
Loaded: 0.000077 seconds

 11387: 0.015089, 0.015089 avg loss, 0.000010 rate, 4.673037 seconds, 364384 images
Loaded: 0.000050 seconds

 11388: 0.009316, 0.014512 avg loss, 0.000010 rate, 4.682217 seconds, 364416 images
Loaded: 0.000044 seconds

 11389: 0.014958, 0.014556 avg loss, 0.000010 rate, 4.681666 seconds, 364448 images
Loaded: 0.000042 seconds

 11390: 0.013107, 0.014411 avg loss, 0.000010 rate, 4.691137 seconds, 364480 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.034616 seconds

 11391: 0.012651, 0.014235 avg loss, 0.000010 rate, 4.519267 seconds, 364512 images
Loaded: 0.000044 seconds

 11392: 0.015499, 0.014362 avg loss, 0.000010 rate, 4.678224 seconds, 364544 images
Loaded: 0.000071 seconds

 11393: 0.018584, 0.014784 avg loss, 0.000010 rate, 4.670307 seconds, 364576 images
Loaded: 0.000049 seconds

 11394: 0.007920, 0.014097 avg loss, 0.000010 rate, 4.678787 seconds, 364608 images
Loaded: 0.000039 seconds

 11395: 0.010280, 0.013716 avg loss, 0.000010 rate, 4.692891 seconds, 364640 images
Loaded: 0.000059 seconds

 11396: 0.015019, 0.013846 avg loss, 0.000010 rate, 4.675093 seconds, 364672 images
Loaded: 0.000070 seconds

 11397: 0.014027, 0.013864 avg loss, 0.000010 rate, 4.676762 seconds, 364704 images
Loaded: 0.000045 seconds

 11398: 0.027325, 0.015210 avg loss, 0.000010 rate, 4.680739 seconds, 364736 images
Loaded: 0.000044 seconds

 11399: 0.020613, 0.015750 avg loss, 0.000010 rate, 4.672027 seconds, 364768 images
Loaded: 0.000071 seconds

 11400: 0.009410, 0.015116 avg loss, 0.000010 rate, 4.687387 seconds, 364800 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.283906 seconds

 11401: 0.016440, 0.015249 avg loss, 0.000010 rate, 3.269589 seconds, 364832 images
Loaded: 0.000072 seconds

 11402: 0.016880, 0.015412 avg loss, 0.000010 rate, 3.384397 seconds, 364864 images
Loaded: 0.000041 seconds

 11403: 0.009026, 0.014773 avg loss, 0.000010 rate, 3.389348 seconds, 364896 images
Loaded: 0.000041 seconds

 11404: 0.014862, 0.014782 avg loss, 0.000010 rate, 3.390150 seconds, 364928 images
Loaded: 0.000064 seconds

 11405: 0.014259, 0.014730 avg loss, 0.000010 rate, 3.395391 seconds, 364960 images
Loaded: 0.000076 seconds

 11406: 0.009129, 0.014170 avg loss, 0.000010 rate, 3.399374 seconds, 364992 images
Loaded: 0.000036 seconds

 11407: 0.007108, 0.013464 avg loss, 0.000010 rate, 3.390184 seconds, 365024 images
Loaded: 0.000041 seconds

 11408: 0.016030, 0.013720 avg loss, 0.000010 rate, 3.394397 seconds, 365056 images
Loaded: 0.000043 seconds

 11409: 0.010317, 0.013380 avg loss, 0.000010 rate, 3.384103 seconds, 365088 images
Loaded: 0.000066 seconds

 11410: 0.009375, 0.012979 avg loss, 0.000010 rate, 3.388049 seconds, 365120 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11411: 0.010225, 0.012704 avg loss, 0.000010 rate, 4.183704 seconds, 365152 images
Loaded: 0.000043 seconds

 11412: 0.018958, 0.013329 avg loss, 0.000010 rate, 4.342360 seconds, 365184 images
Loaded: 0.000068 seconds

 11413: 0.012761, 0.013273 avg loss, 0.000010 rate, 4.326139 seconds, 365216 images
Loaded: 0.000047 seconds

 11414: 0.008948, 0.012840 avg loss, 0.000010 rate, 4.334082 seconds, 365248 images
Loaded: 0.000046 seconds

 11415: 0.012802, 0.012836 avg loss, 0.000010 rate, 4.327679 seconds, 365280 images
Loaded: 0.000036 seconds

 11416: 0.013372, 0.012890 avg loss, 0.000010 rate, 4.332791 seconds, 365312 images
Loaded: 0.000085 seconds

 11417: 0.015247, 0.013126 avg loss, 0.000010 rate, 4.315085 seconds, 365344 images
Loaded: 0.000071 seconds

 11418: 0.013170, 0.013130 avg loss, 0.000010 rate, 4.329001 seconds, 365376 images
Loaded: 0.000067 seconds

 11419: 0.012474, 0.013064 avg loss, 0.000010 rate, 4.319871 seconds, 365408 images
Loaded: 0.000042 seconds

 11420: 0.010130, 0.012771 avg loss, 0.000010 rate, 4.326287 seconds, 365440 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.016775 seconds

 11421: 0.010213, 0.012515 avg loss, 0.000010 rate, 2.704742 seconds, 365472 images
Loaded: 0.000039 seconds

 11422: 0.008985, 0.012162 avg loss, 0.000010 rate, 2.798169 seconds, 365504 images
Loaded: 0.000068 seconds

 11423: 0.009282, 0.011874 avg loss, 0.000010 rate, 2.797748 seconds, 365536 images
Loaded: 0.000039 seconds

 11424: 0.019400, 0.012627 avg loss, 0.000010 rate, 2.801827 seconds, 365568 images
Loaded: 0.000043 seconds

 11425: 0.012452, 0.012609 avg loss, 0.000010 rate, 2.803716 seconds, 365600 images
Loaded: 0.000061 seconds

 11426: 0.010890, 0.012437 avg loss, 0.000010 rate, 2.802695 seconds, 365632 images
Loaded: 0.000044 seconds

 11427: 0.014437, 0.012637 avg loss, 0.000010 rate, 2.802310 seconds, 365664 images
Loaded: 0.000070 seconds

 11428: 0.023004, 0.013674 avg loss, 0.000010 rate, 2.798957 seconds, 365696 images
Loaded: 0.000042 seconds

 11429: 0.010114, 0.013318 avg loss, 0.000010 rate, 2.798906 seconds, 365728 images
Loaded: 0.000071 seconds

 11430: 0.011580, 0.013144 avg loss, 0.000010 rate, 2.802270 seconds, 365760 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.042973 seconds

 11431: 0.011989, 0.013029 avg loss, 0.000010 rate, 2.699037 seconds, 365792 images
Loaded: 0.000042 seconds

 11432: 0.012363, 0.012962 avg loss, 0.000010 rate, 2.805332 seconds, 365824 images
Loaded: 0.000069 seconds

 11433: 0.008521, 0.012518 avg loss, 0.000010 rate, 2.794750 seconds, 365856 images
Loaded: 0.000045 seconds

 11434: 0.011244, 0.012391 avg loss, 0.000010 rate, 2.800978 seconds, 365888 images
Loaded: 0.000044 seconds

 11435: 0.010567, 0.012208 avg loss, 0.000010 rate, 2.801640 seconds, 365920 images
Loaded: 0.000068 seconds

 11436: 0.010093, 0.011997 avg loss, 0.000010 rate, 2.803888 seconds, 365952 images
Loaded: 0.000044 seconds

 11437: 0.007808, 0.011578 avg loss, 0.000010 rate, 2.801574 seconds, 365984 images
Loaded: 0.000077 seconds

 11438: 0.010773, 0.011497 avg loss, 0.000010 rate, 2.795708 seconds, 366016 images
Loaded: 0.000045 seconds

 11439: 0.012078, 0.011555 avg loss, 0.000010 rate, 2.811539 seconds, 366048 images
Loaded: 0.000067 seconds

 11440: 0.010843, 0.011484 avg loss, 0.000010 rate, 2.800494 seconds, 366080 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000051 seconds

 11441: 0.013047, 0.011640 avg loss, 0.000010 rate, 3.014724 seconds, 366112 images
Loaded: 0.000063 seconds

 11442: 0.008741, 0.011350 avg loss, 0.000010 rate, 3.138203 seconds, 366144 images
Loaded: 0.000065 seconds

 11443: 0.008837, 0.011099 avg loss, 0.000010 rate, 3.141295 seconds, 366176 images
Loaded: 0.000066 seconds

 11444: 0.015095, 0.011499 avg loss, 0.000010 rate, 3.142595 seconds, 366208 images
Loaded: 0.000045 seconds

 11445: 0.009493, 0.011298 avg loss, 0.000010 rate, 3.135265 seconds, 366240 images
Loaded: 0.000042 seconds

 11446: 0.005789, 0.010747 avg loss, 0.000010 rate, 3.131664 seconds, 366272 images
Loaded: 0.000043 seconds

 11447: 0.007518, 0.010424 avg loss, 0.000010 rate, 3.146679 seconds, 366304 images
Loaded: 0.000061 seconds

 11448: 0.008770, 0.010259 avg loss, 0.000010 rate, 3.134806 seconds, 366336 images
Loaded: 0.000069 seconds

 11449: 0.010275, 0.010260 avg loss, 0.000010 rate, 3.134153 seconds, 366368 images
Loaded: 0.000043 seconds

 11450: 0.014017, 0.010636 avg loss, 0.000010 rate, 3.140472 seconds, 366400 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.146614 seconds

 11451: 0.011780, 0.010751 avg loss, 0.000010 rate, 2.711304 seconds, 366432 images
Loaded: 0.000051 seconds

 11452: 0.011652, 0.010841 avg loss, 0.000010 rate, 2.797211 seconds, 366464 images
Loaded: 0.000069 seconds

 11453: 0.009168, 0.010673 avg loss, 0.000010 rate, 2.794679 seconds, 366496 images
Loaded: 0.000045 seconds

 11454: 0.012137, 0.010820 avg loss, 0.000010 rate, 2.796786 seconds, 366528 images
Loaded: 0.000034 seconds

 11455: 0.010941, 0.010832 avg loss, 0.000010 rate, 2.798245 seconds, 366560 images
Loaded: 0.000062 seconds

 11456: 0.013547, 0.011103 avg loss, 0.000010 rate, 2.800101 seconds, 366592 images
Loaded: 0.000033 seconds

 11457: 0.014050, 0.011398 avg loss, 0.000010 rate, 2.806337 seconds, 366624 images
Loaded: 0.000099 seconds

 11458: 0.011945, 0.011453 avg loss, 0.000010 rate, 2.803479 seconds, 366656 images
Loaded: 0.000043 seconds

 11459: 0.007921, 0.011100 avg loss, 0.000010 rate, 2.800240 seconds, 366688 images
Loaded: 0.000045 seconds

 11460: 0.010775, 0.011067 avg loss, 0.000010 rate, 2.800801 seconds, 366720 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.113910 seconds

 11461: 0.008926, 0.010853 avg loss, 0.000010 rate, 2.701021 seconds, 366752 images
Loaded: 0.000066 seconds

 11462: 0.017111, 0.011479 avg loss, 0.000010 rate, 2.801768 seconds, 366784 images
Loaded: 0.000057 seconds

 11463: 0.013651, 0.011696 avg loss, 0.000010 rate, 2.807540 seconds, 366816 images
Loaded: 0.000043 seconds

 11464: 0.008125, 0.011339 avg loss, 0.000010 rate, 2.803442 seconds, 366848 images
Loaded: 0.000055 seconds

 11465: 0.010651, 0.011270 avg loss, 0.000010 rate, 2.800692 seconds, 366880 images
Loaded: 0.000048 seconds

 11466: 0.009473, 0.011090 avg loss, 0.000010 rate, 2.804487 seconds, 366912 images
Loaded: 0.000043 seconds

 11467: 0.013459, 0.011327 avg loss, 0.000010 rate, 2.805206 seconds, 366944 images
Loaded: 0.000056 seconds

 11468: 0.008087, 0.011003 avg loss, 0.000010 rate, 2.810541 seconds, 366976 images
Loaded: 0.000064 seconds

 11469: 0.009624, 0.010865 avg loss, 0.000010 rate, 2.802651 seconds, 367008 images
Loaded: 0.000039 seconds

 11470: 0.010078, 0.010787 avg loss, 0.000010 rate, 2.800930 seconds, 367040 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000032 seconds

 11471: 0.023771, 0.012085 avg loss, 0.000010 rate, 1.841398 seconds, 367072 images
Loaded: 0.000073 seconds

 11472: 0.025353, 0.013412 avg loss, 0.000010 rate, 1.921152 seconds, 367104 images
Loaded: 0.000042 seconds

 11473: 0.016500, 0.013721 avg loss, 0.000010 rate, 1.916818 seconds, 367136 images
Loaded: 0.000072 seconds

 11474: 0.016146, 0.013963 avg loss, 0.000010 rate, 1.931489 seconds, 367168 images
Loaded: 0.000043 seconds

 11475: 0.023571, 0.014924 avg loss, 0.000010 rate, 1.929585 seconds, 367200 images
Loaded: 0.000087 seconds

 11476: 0.018179, 0.015250 avg loss, 0.000010 rate, 1.921627 seconds, 367232 images
Loaded: 0.000068 seconds

 11477: 0.008606, 0.014585 avg loss, 0.000010 rate, 1.918945 seconds, 367264 images
Loaded: 0.000067 seconds

 11478: 0.013744, 0.014501 avg loss, 0.000010 rate, 1.922189 seconds, 367296 images
Loaded: 0.000040 seconds

 11479: 0.018582, 0.014909 avg loss, 0.000010 rate, 1.925159 seconds, 367328 images
Loaded: 0.000059 seconds

 11480: 0.015152, 0.014933 avg loss, 0.000010 rate, 1.922394 seconds, 367360 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000039 seconds

 11481: 0.037017, 0.017142 avg loss, 0.000010 rate, 4.489541 seconds, 367392 images
Loaded: 0.000043 seconds

 11482: 0.010995, 0.016527 avg loss, 0.000010 rate, 4.665624 seconds, 367424 images
Loaded: 0.000043 seconds

 11483: 0.021358, 0.017010 avg loss, 0.000010 rate, 4.648344 seconds, 367456 images
Loaded: 0.000071 seconds

 11484: 0.011304, 0.016440 avg loss, 0.000010 rate, 4.659198 seconds, 367488 images
Loaded: 0.000071 seconds

 11485: 0.023309, 0.017126 avg loss, 0.000010 rate, 4.668030 seconds, 367520 images
Loaded: 0.000042 seconds

 11486: 0.016214, 0.017035 avg loss, 0.000010 rate, 4.669665 seconds, 367552 images
Loaded: 0.000069 seconds

 11487: 0.010388, 0.016371 avg loss, 0.000010 rate, 4.673179 seconds, 367584 images
Loaded: 0.000041 seconds

 11488: 0.011143, 0.015848 avg loss, 0.000010 rate, 4.677373 seconds, 367616 images
Loaded: 0.000045 seconds

 11489: 0.007904, 0.015053 avg loss, 0.000010 rate, 4.674208 seconds, 367648 images
Loaded: 0.000045 seconds

 11490: 0.013447, 0.014893 avg loss, 0.000010 rate, 4.682638 seconds, 367680 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.024507 seconds

 11491: 0.012632, 0.014667 avg loss, 0.000010 rate, 2.472973 seconds, 367712 images
Loaded: 0.000066 seconds

 11492: 0.012586, 0.014459 avg loss, 0.000010 rate, 2.567800 seconds, 367744 images
Loaded: 0.000073 seconds

 11493: 0.007101, 0.013723 avg loss, 0.000010 rate, 2.567652 seconds, 367776 images
Loaded: 0.000043 seconds

 11494: 0.011190, 0.013470 avg loss, 0.000010 rate, 2.571231 seconds, 367808 images
Loaded: 0.000068 seconds

 11495: 0.012528, 0.013375 avg loss, 0.000010 rate, 2.566897 seconds, 367840 images
Loaded: 0.000041 seconds

 11496: 0.015031, 0.013541 avg loss, 0.000010 rate, 2.567944 seconds, 367872 images
Loaded: 0.000053 seconds

 11497: 0.011424, 0.013329 avg loss, 0.000010 rate, 2.572080 seconds, 367904 images
Loaded: 0.000073 seconds

 11498: 0.012188, 0.013215 avg loss, 0.000010 rate, 2.569281 seconds, 367936 images
Loaded: 0.000066 seconds

 11499: 0.010438, 0.012937 avg loss, 0.000010 rate, 2.562820 seconds, 367968 images
Loaded: 0.000072 seconds

 11500: 0.011687, 0.012812 avg loss, 0.000010 rate, 2.568784 seconds, 368000 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11501: 0.009632, 0.012494 avg loss, 0.000010 rate, 2.089294 seconds, 368032 images
Loaded: 0.000045 seconds

 11502: 0.010453, 0.012290 avg loss, 0.000010 rate, 2.177127 seconds, 368064 images
Loaded: 0.000069 seconds

 11503: 0.012939, 0.012355 avg loss, 0.000010 rate, 2.170063 seconds, 368096 images
Loaded: 0.000042 seconds

 11504: 0.011183, 0.012238 avg loss, 0.000010 rate, 2.174524 seconds, 368128 images
Loaded: 0.000065 seconds

 11505: 0.014695, 0.012484 avg loss, 0.000010 rate, 2.174614 seconds, 368160 images
Loaded: 0.000063 seconds

 11506: 0.011359, 0.012371 avg loss, 0.000010 rate, 2.169141 seconds, 368192 images
Loaded: 0.000071 seconds

 11507: 0.008842, 0.012018 avg loss, 0.000010 rate, 2.168028 seconds, 368224 images
Loaded: 0.000043 seconds

 11508: 0.012620, 0.012078 avg loss, 0.000010 rate, 2.168058 seconds, 368256 images
Loaded: 0.000065 seconds

 11509: 0.009911, 0.011862 avg loss, 0.000010 rate, 2.172825 seconds, 368288 images
Loaded: 0.000038 seconds

 11510: 0.028860, 0.013561 avg loss, 0.000010 rate, 2.173018 seconds, 368320 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.156334 seconds

 11511: 0.015905, 0.013796 avg loss, 0.000010 rate, 1.845852 seconds, 368352 images
Loaded: 0.000087 seconds

 11512: 0.011315, 0.013548 avg loss, 0.000010 rate, 1.924347 seconds, 368384 images
Loaded: 0.000069 seconds

 11513: 0.012947, 0.013488 avg loss, 0.000010 rate, 1.925541 seconds, 368416 images
Loaded: 0.000072 seconds

 11514: 0.015933, 0.013732 avg loss, 0.000010 rate, 1.929302 seconds, 368448 images
Loaded: 0.000042 seconds

 11515: 0.012525, 0.013611 avg loss, 0.000010 rate, 1.926067 seconds, 368480 images
Loaded: 0.000071 seconds

 11516: 0.017340, 0.013984 avg loss, 0.000010 rate, 1.931949 seconds, 368512 images
Loaded: 0.000073 seconds

 11517: 0.011738, 0.013760 avg loss, 0.000010 rate, 1.921803 seconds, 368544 images
Loaded: 0.000053 seconds

 11518: 0.016223, 0.014006 avg loss, 0.000010 rate, 1.924923 seconds, 368576 images
Loaded: 0.000045 seconds

 11519: 0.018037, 0.014409 avg loss, 0.000010 rate, 1.922825 seconds, 368608 images
Loaded: 0.000050 seconds

 11520: 0.007517, 0.013720 avg loss, 0.000010 rate, 1.917105 seconds, 368640 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 11521: 0.010436, 0.013392 avg loss, 0.000010 rate, 2.087850 seconds, 368672 images
Loaded: 0.000061 seconds

 11522: 0.010642, 0.013117 avg loss, 0.000010 rate, 2.174055 seconds, 368704 images
Loaded: 0.000078 seconds

 11523: 0.012859, 0.013091 avg loss, 0.000010 rate, 2.165313 seconds, 368736 images
Loaded: 0.000067 seconds

 11524: 0.010931, 0.012875 avg loss, 0.000010 rate, 2.163756 seconds, 368768 images
Loaded: 0.000066 seconds

 11525: 0.007958, 0.012383 avg loss, 0.000010 rate, 2.170495 seconds, 368800 images
Loaded: 0.000049 seconds

 11526: 0.010659, 0.012211 avg loss, 0.000010 rate, 2.171460 seconds, 368832 images
Loaded: 0.000045 seconds

 11527: 0.013407, 0.012330 avg loss, 0.000010 rate, 2.169345 seconds, 368864 images
Loaded: 0.000038 seconds

 11528: 0.010809, 0.012178 avg loss, 0.000010 rate, 2.175826 seconds, 368896 images
Loaded: 0.000039 seconds

 11529: 0.017410, 0.012701 avg loss, 0.000010 rate, 2.167558 seconds, 368928 images
Loaded: 0.000063 seconds

 11530: 0.018900, 0.013321 avg loss, 0.000010 rate, 2.179442 seconds, 368960 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000036 seconds

 11531: 0.011168, 0.013106 avg loss, 0.000010 rate, 4.178731 seconds, 368992 images
Loaded: 0.000063 seconds

 11532: 0.006972, 0.012493 avg loss, 0.000010 rate, 4.339324 seconds, 369024 images
Loaded: 0.000048 seconds

 11533: 0.013575, 0.012601 avg loss, 0.000010 rate, 4.313552 seconds, 369056 images
Loaded: 0.000041 seconds

 11534: 0.012196, 0.012560 avg loss, 0.000010 rate, 4.336026 seconds, 369088 images
Loaded: 0.000051 seconds

 11535: 0.011955, 0.012500 avg loss, 0.000010 rate, 4.316668 seconds, 369120 images
Loaded: 0.000070 seconds

 11536: 0.008911, 0.012141 avg loss, 0.000010 rate, 4.345569 seconds, 369152 images
Loaded: 0.000049 seconds

 11537: 0.015759, 0.012503 avg loss, 0.000010 rate, 4.326223 seconds, 369184 images
Loaded: 0.000044 seconds

 11538: 0.010499, 0.012302 avg loss, 0.000010 rate, 4.341238 seconds, 369216 images
Loaded: 0.000045 seconds

 11539: 0.014799, 0.012552 avg loss, 0.000010 rate, 4.330147 seconds, 369248 images
Loaded: 0.000071 seconds

 11540: 0.011315, 0.012428 avg loss, 0.000010 rate, 4.350341 seconds, 369280 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11541: 0.014354, 0.012621 avg loss, 0.000010 rate, 1.866699 seconds, 369312 images
Loaded: 0.000060 seconds

 11542: 0.013726, 0.012731 avg loss, 0.000010 rate, 1.926955 seconds, 369344 images
Loaded: 0.000044 seconds

 11543: 0.016910, 0.013149 avg loss, 0.000010 rate, 1.936897 seconds, 369376 images
Loaded: 0.000040 seconds

 11544: 0.015510, 0.013385 avg loss, 0.000010 rate, 1.927428 seconds, 369408 images
Loaded: 0.000043 seconds

 11545: 0.018302, 0.013877 avg loss, 0.000010 rate, 1.928350 seconds, 369440 images
Loaded: 0.000039 seconds

 11546: 0.010432, 0.013532 avg loss, 0.000010 rate, 1.925469 seconds, 369472 images
Loaded: 0.000046 seconds

 11547: 0.034804, 0.015660 avg loss, 0.000010 rate, 1.930548 seconds, 369504 images
Loaded: 0.000067 seconds

 11548: 0.016658, 0.015759 avg loss, 0.000010 rate, 1.929204 seconds, 369536 images
Loaded: 0.000087 seconds

 11549: 0.017631, 0.015947 avg loss, 0.000010 rate, 1.931805 seconds, 369568 images
Loaded: 0.000069 seconds

 11550: 0.013631, 0.015715 avg loss, 0.000010 rate, 1.927423 seconds, 369600 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11551: 0.015521, 0.015696 avg loss, 0.000010 rate, 3.043593 seconds, 369632 images
Loaded: 0.000043 seconds

 11552: 0.007729, 0.014899 avg loss, 0.000010 rate, 3.136129 seconds, 369664 images
Loaded: 0.000043 seconds

 11553: 0.008426, 0.014252 avg loss, 0.000010 rate, 3.144646 seconds, 369696 images
Loaded: 0.000038 seconds

 11554: 0.015917, 0.014418 avg loss, 0.000010 rate, 3.145740 seconds, 369728 images
Loaded: 0.000047 seconds

 11555: 0.014872, 0.014464 avg loss, 0.000010 rate, 3.143525 seconds, 369760 images
Loaded: 0.000045 seconds

 11556: 0.010971, 0.014114 avg loss, 0.000010 rate, 3.150143 seconds, 369792 images
Loaded: 0.000063 seconds

 11557: 0.012190, 0.013922 avg loss, 0.000010 rate, 3.146728 seconds, 369824 images
Loaded: 0.000076 seconds

 11558: 0.013130, 0.013843 avg loss, 0.000010 rate, 3.147481 seconds, 369856 images
Loaded: 0.000043 seconds

 11559: 0.008735, 0.013332 avg loss, 0.000010 rate, 3.147579 seconds, 369888 images
Loaded: 0.000048 seconds

 11560: 0.011172, 0.013116 avg loss, 0.000010 rate, 3.143918 seconds, 369920 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.026007 seconds

 11561: 0.011477, 0.012952 avg loss, 0.000010 rate, 2.466776 seconds, 369952 images
Loaded: 0.000045 seconds

 11562: 0.013727, 0.013030 avg loss, 0.000010 rate, 2.565197 seconds, 369984 images
Loaded: 0.000044 seconds

 11563: 0.008648, 0.012592 avg loss, 0.000010 rate, 2.559731 seconds, 370016 images
Loaded: 0.000041 seconds

 11564: 0.011000, 0.012432 avg loss, 0.000010 rate, 2.562942 seconds, 370048 images
Loaded: 0.000073 seconds

 11565: 0.010781, 0.012267 avg loss, 0.000010 rate, 2.572246 seconds, 370080 images
Loaded: 0.000068 seconds

 11566: 0.010266, 0.012067 avg loss, 0.000010 rate, 2.555710 seconds, 370112 images
Loaded: 0.000067 seconds

 11567: 0.014596, 0.012320 avg loss, 0.000010 rate, 2.562957 seconds, 370144 images
Loaded: 0.000068 seconds

 11568: 0.011255, 0.012214 avg loss, 0.000010 rate, 2.563278 seconds, 370176 images
Loaded: 0.000067 seconds

 11569: 0.011898, 0.012182 avg loss, 0.000010 rate, 2.565678 seconds, 370208 images
Loaded: 0.000046 seconds

 11570: 0.012035, 0.012167 avg loss, 0.000010 rate, 2.572577 seconds, 370240 images
Resizing
320 x 320 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000039 seconds

 11571: 0.019654, 0.012916 avg loss, 0.000010 rate, 1.619896 seconds, 370272 images
Loaded: 0.000040 seconds

 11572: 0.013889, 0.013013 avg loss, 0.000010 rate, 1.695197 seconds, 370304 images
Loaded: 0.000045 seconds

 11573: 0.023406, 0.014053 avg loss, 0.000010 rate, 1.695310 seconds, 370336 images
Loaded: 0.000058 seconds

 11574: 0.047075, 0.017355 avg loss, 0.000010 rate, 1.695311 seconds, 370368 images
Loaded: 0.000042 seconds

 11575: 0.042971, 0.019916 avg loss, 0.000010 rate, 1.690866 seconds, 370400 images
Loaded: 0.000042 seconds

 11576: 0.018314, 0.019756 avg loss, 0.000010 rate, 1.697280 seconds, 370432 images
Loaded: 0.000050 seconds

 11577: 0.019449, 0.019726 avg loss, 0.000010 rate, 1.709763 seconds, 370464 images
Loaded: 0.000040 seconds

 11578: 0.023708, 0.020124 avg loss, 0.000010 rate, 1.697778 seconds, 370496 images
Loaded: 0.000043 seconds

 11579: 0.052727, 0.023384 avg loss, 0.000010 rate, 1.693245 seconds, 370528 images
Loaded: 0.000041 seconds

 11580: 0.030333, 0.024079 avg loss, 0.000010 rate, 1.696307 seconds, 370560 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11581: 0.009382, 0.022609 avg loss, 0.000010 rate, 3.272996 seconds, 370592 images
Loaded: 0.000041 seconds

 11582: 0.016008, 0.021949 avg loss, 0.000010 rate, 3.396589 seconds, 370624 images
Loaded: 0.000067 seconds

 11583: 0.011752, 0.020930 avg loss, 0.000010 rate, 3.368282 seconds, 370656 images
Loaded: 0.000044 seconds

 11584: 0.011364, 0.019973 avg loss, 0.000010 rate, 3.393946 seconds, 370688 images
Loaded: 0.000058 seconds

 11585: 0.009581, 0.018934 avg loss, 0.000010 rate, 3.377427 seconds, 370720 images
Loaded: 0.000065 seconds

 11586: 0.009736, 0.018014 avg loss, 0.000010 rate, 3.389131 seconds, 370752 images
Loaded: 0.000043 seconds

 11587: 0.010004, 0.017213 avg loss, 0.000010 rate, 3.379314 seconds, 370784 images
Loaded: 0.000042 seconds

 11588: 0.007739, 0.016266 avg loss, 0.000010 rate, 3.397590 seconds, 370816 images
Loaded: 0.000080 seconds

 11589: 0.006381, 0.015277 avg loss, 0.000010 rate, 3.399182 seconds, 370848 images
Loaded: 0.000073 seconds

 11590: 0.010086, 0.014758 avg loss, 0.000010 rate, 3.384293 seconds, 370880 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.026547 seconds

 11591: 0.012202, 0.014502 avg loss, 0.000010 rate, 2.702333 seconds, 370912 images
Loaded: 0.000070 seconds

 11592: 0.013009, 0.014353 avg loss, 0.000010 rate, 2.816645 seconds, 370944 images
Loaded: 0.000044 seconds

 11593: 0.012524, 0.014170 avg loss, 0.000010 rate, 2.783919 seconds, 370976 images
Loaded: 0.000067 seconds

 11594: 0.017195, 0.014473 avg loss, 0.000010 rate, 2.802545 seconds, 371008 images
Loaded: 0.000077 seconds

 11595: 0.009135, 0.013939 avg loss, 0.000010 rate, 2.796558 seconds, 371040 images
Loaded: 0.000056 seconds

 11596: 0.009338, 0.013479 avg loss, 0.000010 rate, 2.810181 seconds, 371072 images
Loaded: 0.000067 seconds

 11597: 0.010758, 0.013207 avg loss, 0.000010 rate, 2.795827 seconds, 371104 images
Loaded: 0.000082 seconds

 11598: 0.016454, 0.013531 avg loss, 0.000010 rate, 2.803026 seconds, 371136 images
Loaded: 0.000037 seconds

 11599: 0.011719, 0.013350 avg loss, 0.000010 rate, 2.792600 seconds, 371168 images
Loaded: 0.000064 seconds

 11600: 0.011263, 0.013141 avg loss, 0.000010 rate, 2.813304 seconds, 371200 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11601: 0.012593, 0.013087 avg loss, 0.000010 rate, 4.179779 seconds, 371232 images
Loaded: 0.000046 seconds

 11602: 0.019458, 0.013724 avg loss, 0.000010 rate, 4.322744 seconds, 371264 images
Loaded: 0.000044 seconds

 11603: 0.009080, 0.013259 avg loss, 0.000010 rate, 4.322312 seconds, 371296 images
Loaded: 0.000065 seconds

 11604: 0.010659, 0.012999 avg loss, 0.000010 rate, 4.337855 seconds, 371328 images
Loaded: 0.000041 seconds

 11605: 0.023964, 0.014096 avg loss, 0.000010 rate, 4.333360 seconds, 371360 images
Loaded: 0.000043 seconds

 11606: 0.020282, 0.014714 avg loss, 0.000010 rate, 4.331117 seconds, 371392 images
Loaded: 0.000046 seconds

 11607: 0.014603, 0.014703 avg loss, 0.000010 rate, 4.335471 seconds, 371424 images
Loaded: 0.000044 seconds

 11608: 0.013050, 0.014538 avg loss, 0.000010 rate, 4.343126 seconds, 371456 images
Loaded: 0.000044 seconds

 11609: 0.010143, 0.014098 avg loss, 0.000010 rate, 4.340648 seconds, 371488 images
Loaded: 0.000046 seconds

 11610: 0.011786, 0.013867 avg loss, 0.000010 rate, 4.356790 seconds, 371520 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000038 seconds

 11611: 0.012758, 0.013756 avg loss, 0.000010 rate, 2.474431 seconds, 371552 images
Loaded: 0.000061 seconds

 11612: 0.011308, 0.013511 avg loss, 0.000010 rate, 2.567231 seconds, 371584 images
Loaded: 0.000046 seconds

 11613: 0.009084, 0.013069 avg loss, 0.000010 rate, 2.573011 seconds, 371616 images
Loaded: 0.000044 seconds

 11614: 0.009918, 0.012754 avg loss, 0.000010 rate, 2.560760 seconds, 371648 images
Loaded: 0.000044 seconds

 11615: 0.010759, 0.012554 avg loss, 0.000010 rate, 2.561842 seconds, 371680 images
Loaded: 0.000069 seconds

 11616: 0.009225, 0.012221 avg loss, 0.000010 rate, 2.570438 seconds, 371712 images
Loaded: 0.000071 seconds

 11617: 0.011799, 0.012179 avg loss, 0.000010 rate, 2.561395 seconds, 371744 images
Loaded: 0.000081 seconds

 11618: 0.011056, 0.012067 avg loss, 0.000010 rate, 2.562175 seconds, 371776 images
Loaded: 0.000041 seconds

 11619: 0.014995, 0.012359 avg loss, 0.000010 rate, 2.563541 seconds, 371808 images
Loaded: 0.000065 seconds

 11620: 0.010394, 0.012163 avg loss, 0.000010 rate, 2.557625 seconds, 371840 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000061 seconds

 11621: 0.016932, 0.012640 avg loss, 0.000010 rate, 3.268486 seconds, 371872 images
Loaded: 0.000041 seconds

 11622: 0.009499, 0.012326 avg loss, 0.000010 rate, 3.390355 seconds, 371904 images
Loaded: 0.000059 seconds

 11623: 0.009969, 0.012090 avg loss, 0.000010 rate, 3.389308 seconds, 371936 images
Loaded: 0.000067 seconds

 11624: 0.013081, 0.012189 avg loss, 0.000010 rate, 3.392202 seconds, 371968 images
Loaded: 0.000068 seconds

 11625: 0.014708, 0.012441 avg loss, 0.000010 rate, 3.393047 seconds, 372000 images
Loaded: 0.000043 seconds

 11626: 0.011153, 0.012312 avg loss, 0.000010 rate, 3.387959 seconds, 372032 images
Loaded: 0.000041 seconds

 11627: 0.010509, 0.012132 avg loss, 0.000010 rate, 3.394285 seconds, 372064 images
Loaded: 0.000041 seconds

 11628: 0.011878, 0.012106 avg loss, 0.000010 rate, 3.384977 seconds, 372096 images
Loaded: 0.000044 seconds

 11629: 0.011283, 0.012024 avg loss, 0.000010 rate, 3.409063 seconds, 372128 images
Loaded: 0.000045 seconds

 11630: 0.025872, 0.013409 avg loss, 0.000010 rate, 3.396146 seconds, 372160 images
Resizing
320 x 320 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11631: 0.078510, 0.019919 avg loss, 0.000010 rate, 1.621259 seconds, 372192 images
Loaded: 0.000044 seconds

 11632: 0.019801, 0.019907 avg loss, 0.000010 rate, 1.692304 seconds, 372224 images
Loaded: 0.000037 seconds

 11633: 0.053191, 0.023236 avg loss, 0.000010 rate, 1.704157 seconds, 372256 images
Loaded: 0.000039 seconds

 11634: 0.027833, 0.023695 avg loss, 0.000010 rate, 1.691060 seconds, 372288 images
Loaded: 0.000067 seconds

 11635: 0.028334, 0.024159 avg loss, 0.000010 rate, 1.698679 seconds, 372320 images
Loaded: 0.000069 seconds

 11636: 0.024864, 0.024230 avg loss, 0.000010 rate, 1.686931 seconds, 372352 images
Loaded: 0.000042 seconds

 11637: 0.084059, 0.030213 avg loss, 0.000010 rate, 1.687478 seconds, 372384 images
Loaded: 0.000041 seconds

 11638: 0.017691, 0.028960 avg loss, 0.000010 rate, 1.689706 seconds, 372416 images
Loaded: 0.000069 seconds

 11639: 0.021980, 0.028262 avg loss, 0.000010 rate, 1.691289 seconds, 372448 images
Loaded: 0.000054 seconds

 11640: 0.014711, 0.026907 avg loss, 0.000010 rate, 1.685961 seconds, 372480 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000038 seconds

 11641: 0.010110, 0.025228 avg loss, 0.000010 rate, 4.500528 seconds, 372512 images
Loaded: 0.000045 seconds

 11642: 0.016731, 0.024378 avg loss, 0.000010 rate, 4.678361 seconds, 372544 images
Loaded: 0.000069 seconds

 11643: 0.019637, 0.023904 avg loss, 0.000010 rate, 4.667305 seconds, 372576 images
Loaded: 0.000073 seconds

 11644: 0.019922, 0.023506 avg loss, 0.000010 rate, 4.669406 seconds, 372608 images
Loaded: 0.000055 seconds

 11645: 0.016029, 0.022758 avg loss, 0.000010 rate, 4.662798 seconds, 372640 images
Loaded: 0.000056 seconds

 11646: 0.011280, 0.021610 avg loss, 0.000010 rate, 4.680256 seconds, 372672 images
Loaded: 0.000043 seconds

 11647: 0.013112, 0.020760 avg loss, 0.000010 rate, 4.676126 seconds, 372704 images
Loaded: 0.000045 seconds

 11648: 0.019100, 0.020594 avg loss, 0.000010 rate, 4.678336 seconds, 372736 images
Loaded: 0.000045 seconds

 11649: 0.007562, 0.019291 avg loss, 0.000010 rate, 4.671119 seconds, 372768 images
Loaded: 0.000073 seconds

 11650: 0.024046, 0.019767 avg loss, 0.000010 rate, 4.667788 seconds, 372800 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 11651: 0.015863, 0.019376 avg loss, 0.000010 rate, 2.094849 seconds, 372832 images
Loaded: 0.000083 seconds

 11652: 0.010659, 0.018504 avg loss, 0.000010 rate, 2.171693 seconds, 372864 images
Loaded: 0.000043 seconds

 11653: 0.057615, 0.022416 avg loss, 0.000010 rate, 2.166024 seconds, 372896 images
Loaded: 0.000036 seconds

 11654: 0.009469, 0.021121 avg loss, 0.000010 rate, 2.167657 seconds, 372928 images
Loaded: 0.000041 seconds

 11655: 0.008962, 0.019905 avg loss, 0.000010 rate, 2.168706 seconds, 372960 images
Loaded: 0.000042 seconds

 11656: 0.011185, 0.019033 avg loss, 0.000010 rate, 2.174624 seconds, 372992 images
Loaded: 0.000064 seconds

 11657: 0.021390, 0.019269 avg loss, 0.000010 rate, 2.163665 seconds, 373024 images
Loaded: 0.000076 seconds

 11658: 0.013223, 0.018664 avg loss, 0.000010 rate, 2.166068 seconds, 373056 images
Loaded: 0.000041 seconds

 11659: 0.013165, 0.018114 avg loss, 0.000010 rate, 2.179226 seconds, 373088 images
Loaded: 0.000055 seconds

 11660: 0.013095, 0.017612 avg loss, 0.000010 rate, 2.163026 seconds, 373120 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11661: 0.007512, 0.016602 avg loss, 0.000010 rate, 3.254262 seconds, 373152 images
Loaded: 0.000042 seconds

 11662: 0.009884, 0.015930 avg loss, 0.000010 rate, 3.376085 seconds, 373184 images
Loaded: 0.000040 seconds

 11663: 0.012559, 0.015593 avg loss, 0.000010 rate, 3.364973 seconds, 373216 images
Loaded: 0.000069 seconds

 11664: 0.019382, 0.015972 avg loss, 0.000010 rate, 3.381871 seconds, 373248 images
Loaded: 0.000065 seconds

 11665: 0.009549, 0.015330 avg loss, 0.000010 rate, 3.376792 seconds, 373280 images
Loaded: 0.000033 seconds

 11666: 0.009685, 0.014765 avg loss, 0.000010 rate, 3.384247 seconds, 373312 images
Loaded: 0.000079 seconds

 11667: 0.014174, 0.014706 avg loss, 0.000010 rate, 3.370610 seconds, 373344 images
Loaded: 0.000045 seconds

 11668: 0.014677, 0.014703 avg loss, 0.000010 rate, 3.381517 seconds, 373376 images
Loaded: 0.000042 seconds

 11669: 0.008421, 0.014075 avg loss, 0.000010 rate, 3.368663 seconds, 373408 images
Loaded: 0.000043 seconds

 11670: 0.008399, 0.013507 avg loss, 0.000010 rate, 3.380451 seconds, 373440 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.036164 seconds

 11671: 0.007864, 0.012943 avg loss, 0.000010 rate, 3.021758 seconds, 373472 images
Loaded: 0.000068 seconds

 11672: 0.010987, 0.012747 avg loss, 0.000010 rate, 3.141005 seconds, 373504 images
Loaded: 0.000067 seconds

 11673: 0.011188, 0.012591 avg loss, 0.000010 rate, 3.132607 seconds, 373536 images
Loaded: 0.000056 seconds

 11674: 0.015296, 0.012862 avg loss, 0.000010 rate, 3.137089 seconds, 373568 images
Loaded: 0.000042 seconds

 11675: 0.009700, 0.012546 avg loss, 0.000010 rate, 3.133887 seconds, 373600 images
Loaded: 0.000079 seconds

 11676: 0.017609, 0.013052 avg loss, 0.000010 rate, 3.135049 seconds, 373632 images
Loaded: 0.000041 seconds

 11677: 0.006869, 0.012434 avg loss, 0.000010 rate, 3.139679 seconds, 373664 images
Loaded: 0.000040 seconds

 11678: 0.015281, 0.012719 avg loss, 0.000010 rate, 3.128919 seconds, 373696 images
Loaded: 0.000071 seconds

 11679: 0.021442, 0.013591 avg loss, 0.000010 rate, 3.137441 seconds, 373728 images
Loaded: 0.000044 seconds

 11680: 0.008357, 0.013067 avg loss, 0.000010 rate, 3.138343 seconds, 373760 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.156112 seconds

 11681: 0.009073, 0.012668 avg loss, 0.000010 rate, 3.011992 seconds, 373792 images
Loaded: 0.000045 seconds

 11682: 0.011072, 0.012508 avg loss, 0.000010 rate, 3.148411 seconds, 373824 images
Loaded: 0.000047 seconds

 11683: 0.016806, 0.012938 avg loss, 0.000010 rate, 3.137047 seconds, 373856 images
Loaded: 0.000041 seconds

 11684: 0.011181, 0.012762 avg loss, 0.000010 rate, 3.145114 seconds, 373888 images
Loaded: 0.000065 seconds

 11685: 0.011216, 0.012608 avg loss, 0.000010 rate, 3.132467 seconds, 373920 images
Loaded: 0.000058 seconds

 11686: 0.011001, 0.012447 avg loss, 0.000010 rate, 3.141583 seconds, 373952 images
Loaded: 0.000045 seconds

 11687: 0.009575, 0.012160 avg loss, 0.000010 rate, 3.126155 seconds, 373984 images
Loaded: 0.000078 seconds

 11688: 0.011050, 0.012049 avg loss, 0.000010 rate, 3.135558 seconds, 374016 images
Loaded: 0.000065 seconds

 11689: 0.007307, 0.011575 avg loss, 0.000010 rate, 3.138392 seconds, 374048 images
Loaded: 0.000062 seconds

 11690: 0.015204, 0.011938 avg loss, 0.000010 rate, 3.143180 seconds, 374080 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.189154 seconds

 11691: 0.009047, 0.011649 avg loss, 0.000010 rate, 2.700530 seconds, 374112 images
Loaded: 0.000043 seconds

 11692: 0.010146, 0.011498 avg loss, 0.000010 rate, 2.803269 seconds, 374144 images
Loaded: 0.000044 seconds

 11693: 0.014251, 0.011774 avg loss, 0.000010 rate, 2.798071 seconds, 374176 images
Loaded: 0.000066 seconds

 11694: 0.011387, 0.011735 avg loss, 0.000010 rate, 2.805207 seconds, 374208 images
Loaded: 0.000044 seconds

 11695: 0.008557, 0.011417 avg loss, 0.000010 rate, 2.809971 seconds, 374240 images
Loaded: 0.000068 seconds

 11696: 0.010329, 0.011308 avg loss, 0.000010 rate, 2.810442 seconds, 374272 images
Loaded: 0.000081 seconds

 11697: 0.007806, 0.010958 avg loss, 0.000010 rate, 2.805580 seconds, 374304 images
Loaded: 0.000044 seconds

 11698: 0.008580, 0.010720 avg loss, 0.000010 rate, 2.811108 seconds, 374336 images
Loaded: 0.000043 seconds

 11699: 0.009050, 0.010553 avg loss, 0.000010 rate, 2.808426 seconds, 374368 images
Loaded: 0.000067 seconds

 11700: 0.008491, 0.010347 avg loss, 0.000010 rate, 2.802147 seconds, 374400 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.177873 seconds

 11701: 0.009134, 0.010226 avg loss, 0.000010 rate, 2.703271 seconds, 374432 images
Loaded: 0.000067 seconds

 11702: 0.010916, 0.010295 avg loss, 0.000010 rate, 2.805494 seconds, 374464 images
Loaded: 0.000035 seconds

 11703: 0.008393, 0.010105 avg loss, 0.000010 rate, 2.802385 seconds, 374496 images
Loaded: 0.000041 seconds

 11704: 0.011192, 0.010213 avg loss, 0.000010 rate, 2.800485 seconds, 374528 images
Loaded: 0.000069 seconds

 11705: 0.008333, 0.010025 avg loss, 0.000010 rate, 2.799452 seconds, 374560 images
Loaded: 0.000041 seconds

 11706: 0.015964, 0.010619 avg loss, 0.000010 rate, 2.802652 seconds, 374592 images
Loaded: 0.000040 seconds

 11707: 0.009933, 0.010551 avg loss, 0.000010 rate, 2.801992 seconds, 374624 images
Loaded: 0.000039 seconds

 11708: 0.011899, 0.010685 avg loss, 0.000010 rate, 2.808078 seconds, 374656 images
Loaded: 0.000039 seconds

 11709: 0.018497, 0.011467 avg loss, 0.000010 rate, 2.806205 seconds, 374688 images
Loaded: 0.000071 seconds

 11710: 0.014296, 0.011749 avg loss, 0.000010 rate, 2.808613 seconds, 374720 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.168300 seconds

 11711: 0.014000, 0.011975 avg loss, 0.000010 rate, 2.701628 seconds, 374752 images
Loaded: 0.000038 seconds

 11712: 0.009775, 0.011755 avg loss, 0.000010 rate, 2.807491 seconds, 374784 images
Loaded: 0.000064 seconds

 11713: 0.011480, 0.011727 avg loss, 0.000010 rate, 2.811733 seconds, 374816 images
Loaded: 0.000046 seconds

 11714: 0.016098, 0.012164 avg loss, 0.000010 rate, 2.806147 seconds, 374848 images
Loaded: 0.000054 seconds

 11715: 0.010227, 0.011971 avg loss, 0.000010 rate, 2.807783 seconds, 374880 images
Loaded: 0.000073 seconds

 11716: 0.010540, 0.011828 avg loss, 0.000010 rate, 2.803523 seconds, 374912 images
Loaded: 0.000042 seconds

 11717: 0.015775, 0.012222 avg loss, 0.000010 rate, 2.804601 seconds, 374944 images
Loaded: 0.000059 seconds

 11718: 0.009037, 0.011904 avg loss, 0.000010 rate, 2.808048 seconds, 374976 images
Loaded: 0.000066 seconds

 11719: 0.010150, 0.011728 avg loss, 0.000010 rate, 2.811423 seconds, 375008 images
Loaded: 0.000072 seconds

 11720: 0.009075, 0.011463 avg loss, 0.000010 rate, 2.808235 seconds, 375040 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000036 seconds

 11721: 0.021006, 0.012417 avg loss, 0.000010 rate, 1.852311 seconds, 375072 images
Loaded: 0.000043 seconds

 11722: 0.022420, 0.013418 avg loss, 0.000010 rate, 1.921913 seconds, 375104 images
Loaded: 0.000063 seconds

 11723: 0.014186, 0.013494 avg loss, 0.000010 rate, 1.919178 seconds, 375136 images
Loaded: 0.000042 seconds

 11724: 0.013684, 0.013513 avg loss, 0.000010 rate, 1.927711 seconds, 375168 images
Loaded: 0.000042 seconds

 11725: 0.014315, 0.013594 avg loss, 0.000010 rate, 1.931649 seconds, 375200 images
Loaded: 0.000069 seconds

 11726: 0.020725, 0.014307 avg loss, 0.000010 rate, 1.925826 seconds, 375232 images
Loaded: 0.000044 seconds

 11727: 0.014095, 0.014286 avg loss, 0.000010 rate, 1.922178 seconds, 375264 images
Loaded: 0.000040 seconds

 11728: 0.022286, 0.015086 avg loss, 0.000010 rate, 1.920885 seconds, 375296 images
Loaded: 0.000053 seconds

 11729: 0.015494, 0.015127 avg loss, 0.000010 rate, 1.928199 seconds, 375328 images
Loaded: 0.000047 seconds

 11730: 0.035549, 0.017169 avg loss, 0.000010 rate, 1.931504 seconds, 375360 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11731: 0.008652, 0.016317 avg loss, 0.000010 rate, 3.266003 seconds, 375392 images
Loaded: 0.000042 seconds

 11732: 0.012420, 0.015927 avg loss, 0.000010 rate, 3.396591 seconds, 375424 images
Loaded: 0.000076 seconds

 11733: 0.009070, 0.015242 avg loss, 0.000010 rate, 3.379475 seconds, 375456 images
Loaded: 0.000068 seconds

 11734: 0.011834, 0.014901 avg loss, 0.000010 rate, 3.391828 seconds, 375488 images
Loaded: 0.000043 seconds

 11735: 0.016399, 0.015051 avg loss, 0.000010 rate, 3.394100 seconds, 375520 images
Loaded: 0.000044 seconds

 11736: 0.010726, 0.014618 avg loss, 0.000010 rate, 3.396630 seconds, 375552 images
Loaded: 0.000069 seconds

 11737: 0.015233, 0.014680 avg loss, 0.000010 rate, 3.381974 seconds, 375584 images
Loaded: 0.000053 seconds

 11738: 0.009227, 0.014134 avg loss, 0.000010 rate, 3.397617 seconds, 375616 images
Loaded: 0.000083 seconds

 11739: 0.013198, 0.014041 avg loss, 0.000010 rate, 3.379986 seconds, 375648 images
Loaded: 0.000042 seconds

 11740: 0.011967, 0.013833 avg loss, 0.000010 rate, 3.393434 seconds, 375680 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.192403 seconds

 11741: 0.025985, 0.015049 avg loss, 0.000010 rate, 2.704744 seconds, 375712 images
Loaded: 0.000070 seconds

 11742: 0.007071, 0.014251 avg loss, 0.000010 rate, 2.813257 seconds, 375744 images
Loaded: 0.000045 seconds

 11743: 0.009947, 0.013820 avg loss, 0.000010 rate, 2.808942 seconds, 375776 images
Loaded: 0.000043 seconds

 11744: 0.011607, 0.013599 avg loss, 0.000010 rate, 2.807005 seconds, 375808 images
Loaded: 0.000067 seconds

 11745: 0.009040, 0.013143 avg loss, 0.000010 rate, 2.808993 seconds, 375840 images
Loaded: 0.000034 seconds

 11746: 0.008585, 0.012687 avg loss, 0.000010 rate, 2.803779 seconds, 375872 images
Loaded: 0.000084 seconds

 11747: 0.011328, 0.012551 avg loss, 0.000010 rate, 2.813780 seconds, 375904 images
Loaded: 0.000041 seconds

 11748: 0.007512, 0.012047 avg loss, 0.000010 rate, 2.808472 seconds, 375936 images
Loaded: 0.000046 seconds

 11749: 0.013113, 0.012154 avg loss, 0.000010 rate, 2.806551 seconds, 375968 images
Loaded: 0.000044 seconds

 11750: 0.009291, 0.011868 avg loss, 0.000010 rate, 2.803785 seconds, 376000 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000062 seconds

 11751: 0.016312, 0.012312 avg loss, 0.000010 rate, 3.032354 seconds, 376032 images
Loaded: 0.000040 seconds

 11752: 0.009388, 0.012020 avg loss, 0.000010 rate, 3.126580 seconds, 376064 images
Loaded: 0.000069 seconds

 11753: 0.014804, 0.012298 avg loss, 0.000010 rate, 3.139225 seconds, 376096 images
Loaded: 0.000039 seconds

 11754: 0.015137, 0.012582 avg loss, 0.000010 rate, 3.135141 seconds, 376128 images
Loaded: 0.000035 seconds

 11755: 0.011449, 0.012469 avg loss, 0.000010 rate, 3.140814 seconds, 376160 images
Loaded: 0.000044 seconds

 11756: 0.009259, 0.012148 avg loss, 0.000010 rate, 3.136463 seconds, 376192 images
Loaded: 0.000045 seconds

 11757: 0.010837, 0.012017 avg loss, 0.000010 rate, 3.130326 seconds, 376224 images
Loaded: 0.000043 seconds

 11758: 0.014759, 0.012291 avg loss, 0.000010 rate, 3.143077 seconds, 376256 images
Loaded: 0.000045 seconds

 11759: 0.012606, 0.012322 avg loss, 0.000010 rate, 3.144677 seconds, 376288 images
Loaded: 0.000056 seconds

 11760: 0.007821, 0.011872 avg loss, 0.000010 rate, 3.134795 seconds, 376320 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11761: 0.010518, 0.011737 avg loss, 0.000010 rate, 4.483881 seconds, 376352 images
Loaded: 0.000066 seconds

 11762: 0.014869, 0.012050 avg loss, 0.000010 rate, 4.651004 seconds, 376384 images
Loaded: 0.000064 seconds

 11763: 0.016640, 0.012509 avg loss, 0.000010 rate, 4.656960 seconds, 376416 images
Loaded: 0.000069 seconds

 11764: 0.009226, 0.012181 avg loss, 0.000010 rate, 4.665759 seconds, 376448 images
Loaded: 0.000048 seconds

 11765: 0.014632, 0.012426 avg loss, 0.000010 rate, 4.664320 seconds, 376480 images
Loaded: 0.000043 seconds

 11766: 0.011945, 0.012378 avg loss, 0.000010 rate, 4.688074 seconds, 376512 images
Loaded: 0.000044 seconds

 11767: 0.010655, 0.012206 avg loss, 0.000010 rate, 4.685913 seconds, 376544 images
Loaded: 0.000066 seconds

 11768: 0.022263, 0.013211 avg loss, 0.000010 rate, 4.691359 seconds, 376576 images
Loaded: 0.000071 seconds

 11769: 0.013094, 0.013200 avg loss, 0.000010 rate, 4.669655 seconds, 376608 images
Loaded: 0.000075 seconds

 11770: 0.010375, 0.012917 avg loss, 0.000010 rate, 4.670244 seconds, 376640 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000037 seconds

 11771: 0.015042, 0.013130 avg loss, 0.000010 rate, 4.180371 seconds, 376672 images
Loaded: 0.000044 seconds

 11772: 0.009097, 0.012726 avg loss, 0.000010 rate, 4.329058 seconds, 376704 images
Loaded: 0.000046 seconds

 11773: 0.013918, 0.012846 avg loss, 0.000010 rate, 4.325349 seconds, 376736 images
Loaded: 0.000042 seconds

 11774: 0.012798, 0.012841 avg loss, 0.000010 rate, 4.335592 seconds, 376768 images
Loaded: 0.000044 seconds

 11775: 0.019342, 0.013491 avg loss, 0.000010 rate, 4.319577 seconds, 376800 images
Loaded: 0.000045 seconds

 11776: 0.014319, 0.013574 avg loss, 0.000010 rate, 4.324477 seconds, 376832 images
Loaded: 0.000047 seconds

 11777: 0.012893, 0.013506 avg loss, 0.000010 rate, 4.318743 seconds, 376864 images
Loaded: 0.000040 seconds

 11778: 0.013862, 0.013541 avg loss, 0.000010 rate, 4.332190 seconds, 376896 images
Loaded: 0.000088 seconds

 11779: 0.010557, 0.013243 avg loss, 0.000010 rate, 4.322406 seconds, 376928 images
Loaded: 0.000075 seconds

 11780: 0.011061, 0.013025 avg loss, 0.000010 rate, 4.319292 seconds, 376960 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.064042 seconds

 11781: 0.012509, 0.012973 avg loss, 0.000010 rate, 4.172589 seconds, 376992 images
Loaded: 0.000078 seconds

 11782: 0.023442, 0.014020 avg loss, 0.000010 rate, 4.320068 seconds, 377024 images
Loaded: 0.000073 seconds

 11783: 0.011346, 0.013753 avg loss, 0.000010 rate, 4.300900 seconds, 377056 images
Loaded: 0.000039 seconds

 11784: 0.011046, 0.013482 avg loss, 0.000010 rate, 4.314272 seconds, 377088 images
Loaded: 0.000057 seconds

 11785: 0.017660, 0.013900 avg loss, 0.000010 rate, 4.306539 seconds, 377120 images
Loaded: 0.000045 seconds

 11786: 0.015185, 0.014028 avg loss, 0.000010 rate, 4.315120 seconds, 377152 images
Loaded: 0.000077 seconds

 11787: 0.016359, 0.014261 avg loss, 0.000010 rate, 4.310257 seconds, 377184 images
Loaded: 0.000054 seconds

 11788: 0.013643, 0.014200 avg loss, 0.000010 rate, 4.323283 seconds, 377216 images
Loaded: 0.000079 seconds

 11789: 0.008912, 0.013671 avg loss, 0.000010 rate, 4.306254 seconds, 377248 images
Loaded: 0.000043 seconds

 11790: 0.012102, 0.013514 avg loss, 0.000010 rate, 4.331338 seconds, 377280 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000043 seconds

 11791: 0.011508, 0.013313 avg loss, 0.000010 rate, 2.083750 seconds, 377312 images
Loaded: 0.000061 seconds

 11792: 0.009356, 0.012918 avg loss, 0.000010 rate, 2.168514 seconds, 377344 images
Loaded: 0.000043 seconds

 11793: 0.011269, 0.012753 avg loss, 0.000010 rate, 2.169447 seconds, 377376 images
Loaded: 0.000055 seconds

 11794: 0.015114, 0.012989 avg loss, 0.000010 rate, 2.168441 seconds, 377408 images
Loaded: 0.000064 seconds

 11795: 0.013399, 0.013030 avg loss, 0.000010 rate, 2.161440 seconds, 377440 images
Loaded: 0.000043 seconds

 11796: 0.014124, 0.013139 avg loss, 0.000010 rate, 2.165224 seconds, 377472 images
Loaded: 0.000067 seconds

 11797: 0.025700, 0.014395 avg loss, 0.000010 rate, 2.168908 seconds, 377504 images
Loaded: 0.000066 seconds

 11798: 0.014147, 0.014371 avg loss, 0.000010 rate, 2.168993 seconds, 377536 images
Loaded: 0.000041 seconds

 11799: 0.012171, 0.014151 avg loss, 0.000010 rate, 2.169400 seconds, 377568 images
Loaded: 0.000041 seconds

 11800: 0.010858, 0.013821 avg loss, 0.000010 rate, 2.173941 seconds, 377600 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.267218 seconds

 11801: 0.015586, 0.013998 avg loss, 0.000010 rate, 1.842929 seconds, 377632 images
Loaded: 0.000043 seconds

 11802: 0.027318, 0.015330 avg loss, 0.000010 rate, 1.921968 seconds, 377664 images
Loaded: 0.000040 seconds

 11803: 0.015754, 0.015372 avg loss, 0.000010 rate, 1.919630 seconds, 377696 images
Loaded: 0.000062 seconds

 11804: 0.014526, 0.015288 avg loss, 0.000010 rate, 1.921936 seconds, 377728 images
Loaded: 0.000068 seconds

 11805: 0.017568, 0.015516 avg loss, 0.000010 rate, 1.916892 seconds, 377760 images
Loaded: 0.000065 seconds

 11806: 0.018720, 0.015836 avg loss, 0.000010 rate, 1.928039 seconds, 377792 images
Loaded: 0.000041 seconds

 11807: 0.036886, 0.017941 avg loss, 0.000010 rate, 1.934419 seconds, 377824 images
Loaded: 0.000038 seconds

 11808: 0.018345, 0.017981 avg loss, 0.000010 rate, 1.924530 seconds, 377856 images
Loaded: 0.000041 seconds

 11809: 0.016722, 0.017856 avg loss, 0.000010 rate, 1.930563 seconds, 377888 images
Loaded: 0.000052 seconds

 11810: 0.017350, 0.017805 avg loss, 0.000010 rate, 1.919739 seconds, 377920 images
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000039 seconds

 11811: 0.010847, 0.017109 avg loss, 0.000010 rate, 2.481968 seconds, 377952 images
Loaded: 0.000062 seconds

 11812: 0.018117, 0.017210 avg loss, 0.000010 rate, 2.560257 seconds, 377984 images
Loaded: 0.000041 seconds

 11813: 0.009630, 0.016452 avg loss, 0.000010 rate, 2.566039 seconds, 378016 images
Loaded: 0.000041 seconds

 11814: 0.010794, 0.015886 avg loss, 0.000010 rate, 2.564982 seconds, 378048 images
Loaded: 0.000044 seconds

 11815: 0.011923, 0.015490 avg loss, 0.000010 rate, 2.562215 seconds, 378080 images
Loaded: 0.000041 seconds

 11816: 0.011823, 0.015123 avg loss, 0.000010 rate, 2.566416 seconds, 378112 images
Loaded: 0.000043 seconds

 11817: 0.012636, 0.014875 avg loss, 0.000010 rate, 2.572663 seconds, 378144 images
Loaded: 0.000043 seconds

 11818: 0.013107, 0.014698 avg loss, 0.000010 rate, 2.570526 seconds, 378176 images
Loaded: 0.000043 seconds

 11819: 0.010218, 0.014250 avg loss, 0.000010 rate, 2.567993 seconds, 378208 images
Loaded: 0.000070 seconds

 11820: 0.016429, 0.014468 avg loss, 0.000010 rate, 2.569802 seconds, 378240 images
Resizing
320 x 320 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.110239 seconds

 11821: 0.016080, 0.014629 avg loss, 0.000010 rate, 1.626505 seconds, 378272 images
Loaded: 0.000041 seconds

 11822: 0.031637, 0.016330 avg loss, 0.000010 rate, 1.697104 seconds, 378304 images
Loaded: 0.000067 seconds

 11823: 0.022557, 0.016952 avg loss, 0.000010 rate, 1.699173 seconds, 378336 images
Loaded: 0.000080 seconds

 11824: 0.023720, 0.017629 avg loss, 0.000010 rate, 1.698037 seconds, 378368 images
Loaded: 0.000042 seconds

 11825: 0.021455, 0.018012 avg loss, 0.000010 rate, 1.690133 seconds, 378400 images
Loaded: 0.000042 seconds

 11826: 0.021846, 0.018395 avg loss, 0.000010 rate, 1.690067 seconds, 378432 images
Loaded: 0.000041 seconds

 11827: 0.020884, 0.018644 avg loss, 0.000010 rate, 1.693163 seconds, 378464 images
Loaded: 0.000066 seconds

 11828: 0.033213, 0.020101 avg loss, 0.000010 rate, 1.706280 seconds, 378496 images
Loaded: 0.000068 seconds

 11829: 0.099292, 0.028020 avg loss, 0.000010 rate, 1.696244 seconds, 378528 images
Loaded: 0.000041 seconds

 11830: 0.030567, 0.028275 avg loss, 0.000010 rate, 1.693593 seconds, 378560 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000042 seconds

 11831: 0.016885, 0.027136 avg loss, 0.000010 rate, 4.502745 seconds, 378592 images
Loaded: 0.000076 seconds

 11832: 0.010485, 0.025471 avg loss, 0.000010 rate, 4.660047 seconds, 378624 images
Loaded: 0.000073 seconds

 11833: 0.014318, 0.024355 avg loss, 0.000010 rate, 4.651460 seconds, 378656 images
Loaded: 0.000048 seconds

 11834: 0.041634, 0.026083 avg loss, 0.000010 rate, 4.661076 seconds, 378688 images
Loaded: 0.000054 seconds

 11835: 0.021932, 0.025668 avg loss, 0.000010 rate, 4.659890 seconds, 378720 images
Loaded: 0.000071 seconds

 11836: 0.024461, 0.025547 avg loss, 0.000010 rate, 4.675352 seconds, 378752 images
Loaded: 0.000072 seconds

 11837: 0.014892, 0.024482 avg loss, 0.000010 rate, 4.644701 seconds, 378784 images
Loaded: 0.000042 seconds

 11838: 0.015064, 0.023540 avg loss, 0.000010 rate, 4.673531 seconds, 378816 images
Loaded: 0.000045 seconds

 11839: 0.015584, 0.022744 avg loss, 0.000010 rate, 4.648880 seconds, 378848 images
Loaded: 0.000064 seconds

 11840: 0.015938, 0.022064 avg loss, 0.000010 rate, 4.670270 seconds, 378880 images
Resizing
608 x 608 
 try to allocate additional workspace_size = 108.69 MB 
 CUDA allocate done! 
Loaded: 0.000220 seconds

 11841: 0.013208, 0.021178 avg loss, 0.000010 rate, 4.935686 seconds, 378912 images
Loaded: 0.000076 seconds

 11842: 0.017978, 0.020858 avg loss, 0.000010 rate, 5.094953 seconds, 378944 images
Loaded: 0.000041 seconds

 11843: 0.019658, 0.020738 avg loss, 0.000010 rate, 5.102595 seconds, 378976 images
Loaded: 0.000042 seconds

 11844: 0.015915, 0.020256 avg loss, 0.000010 rate, 5.116146 seconds, 379008 images
Loaded: 0.000065 seconds

 11845: 0.010040, 0.019234 avg loss, 0.000010 rate, 5.114908 seconds, 379040 images
Loaded: 0.000039 seconds

 11846: 0.030615, 0.020372 avg loss, 0.000010 rate, 5.127734 seconds, 379072 images
Loaded: 0.000056 seconds

 11847: 0.018788, 0.020214 avg loss, 0.000010 rate, 5.142722 seconds, 379104 images
Loaded: 0.000068 seconds

 11848: 0.016065, 0.019799 avg loss, 0.000010 rate, 5.137747 seconds, 379136 images
Loaded: 0.000044 seconds

 11849: 0.010439, 0.018863 avg loss, 0.000010 rate, 5.144847 seconds, 379168 images
Loaded: 0.000041 seconds

 11850: 0.016349, 0.018612 avg loss, 0.000010 rate, 5.137250 seconds, 379200 images
Resizing
512 x 512 
 try to allocate additional workspace_size = 77.07 MB 
 CUDA allocate done! 
Loaded: 0.012071 seconds

 11851: 0.017566, 0.018507 avg loss, 0.000010 rate, 3.267668 seconds, 379232 images
Loaded: 0.000069 seconds

 11852: 0.012614, 0.017918 avg loss, 0.000010 rate, 3.404790 seconds, 379264 images
Loaded: 0.000052 seconds

 11853: 0.014527, 0.017579 avg loss, 0.000010 rate, 3.401145 seconds, 379296 images
Loaded: 0.000067 seconds

 11854: 0.011417, 0.016962 avg loss, 0.000010 rate, 3.399360 seconds, 379328 images
Loaded: 0.000045 seconds

 11855: 0.010054, 0.016272 avg loss, 0.000010 rate, 3.391368 seconds, 379360 images
Loaded: 0.000039 seconds

 11856: 0.009865, 0.015631 avg loss, 0.000010 rate, 3.384841 seconds, 379392 images
Loaded: 0.000073 seconds

 11857: 0.010285, 0.015096 avg loss, 0.000010 rate, 3.394503 seconds, 379424 images
Loaded: 0.000043 seconds

 11858: 0.010651, 0.014652 avg loss, 0.000010 rate, 3.391703 seconds, 379456 images
Loaded: 0.000039 seconds

 11859: 0.013239, 0.014510 avg loss, 0.000010 rate, 3.389797 seconds, 379488 images
Loaded: 0.000073 seconds

 11860: 0.010456, 0.014105 avg loss, 0.000010 rate, 3.391163 seconds, 379520 images
Resizing
608 x 608 
 try to allocate additional workspace_size = 108.69 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11861: 0.039683, 0.016663 avg loss, 0.000010 rate, 4.923000 seconds, 379552 images
Loaded: 0.000070 seconds

 11862: 0.011236, 0.016120 avg loss, 0.000010 rate, 5.099344 seconds, 379584 images
Loaded: 0.000064 seconds

 11863: 0.018588, 0.016367 avg loss, 0.000010 rate, 5.111735 seconds, 379616 images
Loaded: 0.000034 seconds

 11864: 0.037407, 0.018471 avg loss, 0.000010 rate, 5.093630 seconds, 379648 images
Loaded: 0.000034 seconds

 11865: 0.030348, 0.019659 avg loss, 0.000010 rate, 5.094407 seconds, 379680 images
Loaded: 0.000039 seconds

 11866: 0.018232, 0.019516 avg loss, 0.000010 rate, 5.102373 seconds, 379712 images
Loaded: 0.000046 seconds

 11867: 0.015677, 0.019132 avg loss, 0.000010 rate, 5.098526 seconds, 379744 images
Loaded: 0.000046 seconds

 11868: 0.017389, 0.018958 avg loss, 0.000010 rate, 5.096274 seconds, 379776 images
Loaded: 0.000069 seconds

 11869: 0.018389, 0.018901 avg loss, 0.000010 rate, 5.081300 seconds, 379808 images
Loaded: 0.000051 seconds

 11870: 0.011672, 0.018178 avg loss, 0.000010 rate, 5.099425 seconds, 379840 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.012055 seconds

 11871: 0.006931, 0.017053 avg loss, 0.000010 rate, 4.163983 seconds, 379872 images
Loaded: 0.000051 seconds

 11872: 0.010934, 0.016441 avg loss, 0.000010 rate, 4.313187 seconds, 379904 images
Loaded: 0.000068 seconds

 11873: 0.010366, 0.015834 avg loss, 0.000010 rate, 4.314559 seconds, 379936 images
Loaded: 0.000041 seconds

 11874: 0.010448, 0.015295 avg loss, 0.000010 rate, 4.316227 seconds, 379968 images
Loaded: 0.000044 seconds

 11875: 0.010914, 0.014857 avg loss, 0.000010 rate, 4.327328 seconds, 380000 images
Loaded: 0.000067 seconds

 11876: 0.010100, 0.014381 avg loss, 0.000010 rate, 4.319618 seconds, 380032 images
Loaded: 0.000042 seconds

 11877: 0.009987, 0.013942 avg loss, 0.000010 rate, 4.318505 seconds, 380064 images
Loaded: 0.000042 seconds

 11878: 0.012497, 0.013797 avg loss, 0.000010 rate, 4.329213 seconds, 380096 images
Loaded: 0.000062 seconds

 11879: 0.009421, 0.013360 avg loss, 0.000010 rate, 4.311142 seconds, 380128 images
Loaded: 0.000068 seconds

 11880: 0.012026, 0.013226 avg loss, 0.000010 rate, 4.319567 seconds, 380160 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.000038 seconds

 11881: 0.008939, 0.012798 avg loss, 0.000010 rate, 2.704055 seconds, 380192 images
Loaded: 0.000044 seconds

 11882: 0.008439, 0.012362 avg loss, 0.000010 rate, 2.797547 seconds, 380224 images
Loaded: 0.000039 seconds

 11883: 0.010836, 0.012209 avg loss, 0.000010 rate, 2.808314 seconds, 380256 images
Loaded: 0.000045 seconds

 11884: 0.008631, 0.011851 avg loss, 0.000010 rate, 2.802441 seconds, 380288 images
Loaded: 0.000044 seconds

 11885: 0.009721, 0.011638 avg loss, 0.000010 rate, 2.803059 seconds, 380320 images
Loaded: 0.000070 seconds

 11886: 0.026277, 0.013102 avg loss, 0.000010 rate, 2.801663 seconds, 380352 images
Loaded: 0.000056 seconds

 11887: 0.013560, 0.013148 avg loss, 0.000010 rate, 2.810754 seconds, 380384 images
Loaded: 0.000039 seconds

 11888: 0.012367, 0.013070 avg loss, 0.000010 rate, 2.814589 seconds, 380416 images
Loaded: 0.000070 seconds

 11889: 0.007656, 0.012529 avg loss, 0.000010 rate, 2.804157 seconds, 380448 images
Loaded: 0.000046 seconds

 11890: 0.013871, 0.012663 avg loss, 0.000010 rate, 2.803575 seconds, 380480 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.058841 seconds

 11891: 0.015504, 0.012947 avg loss, 0.000010 rate, 1.849005 seconds, 380512 images
Loaded: 0.000072 seconds

 11892: 0.013341, 0.012986 avg loss, 0.000010 rate, 1.926241 seconds, 380544 images
Loaded: 0.000041 seconds

 11893: 0.016252, 0.013313 avg loss, 0.000010 rate, 1.930193 seconds, 380576 images
Loaded: 0.000042 seconds

 11894: 0.014605, 0.013442 avg loss, 0.000010 rate, 1.923767 seconds, 380608 images
Loaded: 0.000071 seconds

 11895: 0.020145, 0.014112 avg loss, 0.000010 rate, 1.942803 seconds, 380640 images
Loaded: 0.000075 seconds

 11896: 0.014671, 0.014168 avg loss, 0.000010 rate, 1.925507 seconds, 380672 images
Loaded: 0.000037 seconds

 11897: 0.013566, 0.014108 avg loss, 0.000010 rate, 1.933634 seconds, 380704 images
Loaded: 0.000040 seconds

 11898: 0.011440, 0.013841 avg loss, 0.000010 rate, 1.928382 seconds, 380736 images
Loaded: 0.000041 seconds

 11899: 0.015493, 0.014007 avg loss, 0.000010 rate, 1.925585 seconds, 380768 images
Loaded: 0.000042 seconds

 11900: 0.011991, 0.013805 avg loss, 0.000010 rate, 1.929240 seconds, 380800 images
Saving weights to backup/custom-train-yolo_last.weights
Resizing
416 x 416 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000044 seconds

 11901: 0.012002, 0.013625 avg loss, 0.000010 rate, 2.466217 seconds, 380832 images
Loaded: 0.000042 seconds

 11902: 0.009148, 0.013177 avg loss, 0.000010 rate, 2.567199 seconds, 380864 images
Loaded: 0.000043 seconds

 11903: 0.010845, 0.012944 avg loss, 0.000010 rate, 2.565445 seconds, 380896 images
Loaded: 0.000040 seconds

 11904: 0.013794, 0.013029 avg loss, 0.000010 rate, 2.568492 seconds, 380928 images
Loaded: 0.000045 seconds

 11905: 0.012612, 0.012987 avg loss, 0.000010 rate, 2.567266 seconds, 380960 images
Loaded: 0.000069 seconds

 11906: 0.009583, 0.012647 avg loss, 0.000010 rate, 2.560696 seconds, 380992 images
Loaded: 0.000086 seconds

 11907: 0.010255, 0.012408 avg loss, 0.000010 rate, 2.564308 seconds, 381024 images
Loaded: 0.000081 seconds

 11908: 0.009909, 0.012158 avg loss, 0.000010 rate, 2.558609 seconds, 381056 images
Loaded: 0.000077 seconds

 11909: 0.008760, 0.011818 avg loss, 0.000010 rate, 2.562863 seconds, 381088 images
Loaded: 0.000040 seconds

 11910: 0.012718, 0.011908 avg loss, 0.000010 rate, 2.563730 seconds, 381120 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.000037 seconds

 11911: 0.010150, 0.011732 avg loss, 0.000010 rate, 4.500326 seconds, 381152 images
Loaded: 0.000080 seconds

 11912: 0.017664, 0.012325 avg loss, 0.000010 rate, 4.673773 seconds, 381184 images
Loaded: 0.000049 seconds

 11913: 0.012544, 0.012347 avg loss, 0.000010 rate, 4.676475 seconds, 381216 images
Loaded: 0.000081 seconds

 11914: 0.019635, 0.013076 avg loss, 0.000010 rate, 4.682135 seconds, 381248 images
Loaded: 0.000093 seconds

 11915: 0.014783, 0.013247 avg loss, 0.000010 rate, 4.678580 seconds, 381280 images
Loaded: 0.000068 seconds

 11916: 0.013498, 0.013272 avg loss, 0.000010 rate, 4.689656 seconds, 381312 images
Loaded: 0.000043 seconds

 11917: 0.019614, 0.013906 avg loss, 0.000010 rate, 4.688869 seconds, 381344 images
Loaded: 0.000044 seconds

 11918: 0.010740, 0.013589 avg loss, 0.000010 rate, 4.681756 seconds, 381376 images
Loaded: 0.000066 seconds

 11919: 0.029779, 0.015208 avg loss, 0.000010 rate, 4.683551 seconds, 381408 images
Loaded: 0.000069 seconds

 11920: 0.010041, 0.014692 avg loss, 0.000010 rate, 4.680050 seconds, 381440 images
Resizing
576 x 576 
 try to allocate additional workspace_size = 97.55 MB 
 CUDA allocate done! 
Loaded: 0.029241 seconds

 11921: 0.012725, 0.014495 avg loss, 0.000010 rate, 4.518601 seconds, 381472 images
Loaded: 0.000044 seconds

 11922: 0.010633, 0.014109 avg loss, 0.000010 rate, 4.668431 seconds, 381504 images
Loaded: 0.000073 seconds

 11923: 0.022676, 0.014966 avg loss, 0.000010 rate, 4.672527 seconds, 381536 images
Loaded: 0.000043 seconds

 11924: 0.016512, 0.015120 avg loss, 0.000010 rate, 4.668491 seconds, 381568 images
Loaded: 0.000047 seconds

 11925: 0.015457, 0.015154 avg loss, 0.000010 rate, 4.671771 seconds, 381600 images
Loaded: 0.000073 seconds

 11926: 0.018843, 0.015523 avg loss, 0.000010 rate, 4.664516 seconds, 381632 images
Loaded: 0.000068 seconds

 11927: 0.009574, 0.014928 avg loss, 0.000010 rate, 4.657794 seconds, 381664 images
Loaded: 0.000044 seconds

 11928: 0.011582, 0.014593 avg loss, 0.000010 rate, 4.660821 seconds, 381696 images
Loaded: 0.000039 seconds

 11929: 0.014773, 0.014611 avg loss, 0.000010 rate, 4.655566 seconds, 381728 images
Loaded: 0.000043 seconds

 11930: 0.018141, 0.014964 avg loss, 0.000010 rate, 4.662284 seconds, 381760 images
Resizing
352 x 352 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000040 seconds

 11931: 0.014574, 0.014925 avg loss, 0.000010 rate, 1.842887 seconds, 381792 images
Loaded: 0.000043 seconds

 11932: 0.016585, 0.015091 avg loss, 0.000010 rate, 1.922143 seconds, 381824 images
Loaded: 0.000045 seconds

 11933: 0.019225, 0.015505 avg loss, 0.000010 rate, 1.920419 seconds, 381856 images
Loaded: 0.000068 seconds

 11934: 0.011518, 0.015106 avg loss, 0.000010 rate, 1.924662 seconds, 381888 images
Loaded: 0.000066 seconds

 11935: 0.014164, 0.015012 avg loss, 0.000010 rate, 1.921433 seconds, 381920 images
Loaded: 0.000042 seconds

 11936: 0.018200, 0.015331 avg loss, 0.000010 rate, 1.919550 seconds, 381952 images
Loaded: 0.000041 seconds

 11937: 0.013347, 0.015132 avg loss, 0.000010 rate, 1.916653 seconds, 381984 images
Loaded: 0.000075 seconds

 11938: 0.016207, 0.015240 avg loss, 0.000010 rate, 1.928937 seconds, 382016 images
Loaded: 0.000040 seconds

 11939: 0.020633, 0.015779 avg loss, 0.000010 rate, 1.920882 seconds, 382048 images
Loaded: 0.000043 seconds

 11940: 0.025663, 0.016767 avg loss, 0.000010 rate, 1.930680 seconds, 382080 images
Resizing
448 x 448 
 try to allocate additional workspace_size = 59.01 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11941: 0.008363, 0.015927 avg loss, 0.000010 rate, 2.706278 seconds, 382112 images
Loaded: 0.000067 seconds

 11942: 0.011005, 0.015435 avg loss, 0.000010 rate, 2.799379 seconds, 382144 images
Loaded: 0.000065 seconds

 11943: 0.015673, 0.015459 avg loss, 0.000010 rate, 2.805115 seconds, 382176 images
Loaded: 0.000044 seconds

 11944: 0.008536, 0.014766 avg loss, 0.000010 rate, 2.802071 seconds, 382208 images
Loaded: 0.000048 seconds

 11945: 0.009702, 0.014260 avg loss, 0.000010 rate, 2.809662 seconds, 382240 images
Loaded: 0.000038 seconds

 11946: 0.011489, 0.013983 avg loss, 0.000010 rate, 2.796315 seconds, 382272 images
Loaded: 0.000068 seconds

 11947: 0.010730, 0.013658 avg loss, 0.000010 rate, 2.799646 seconds, 382304 images
Loaded: 0.000041 seconds

 11948: 0.009304, 0.013222 avg loss, 0.000010 rate, 2.805930 seconds, 382336 images
Loaded: 0.000069 seconds

 11949: 0.012682, 0.013168 avg loss, 0.000010 rate, 2.816619 seconds, 382368 images
Loaded: 0.000046 seconds

 11950: 0.009084, 0.012760 avg loss, 0.000010 rate, 2.801812 seconds, 382400 images
Resizing
480 x 480 
 try to allocate additional workspace_size = 67.74 MB 
 CUDA allocate done! 
Loaded: 0.000041 seconds

 11951: 0.006227, 0.012106 avg loss, 0.000010 rate, 3.017997 seconds, 382432 images
Loaded: 0.000061 seconds

 11952: 0.010696, 0.011965 avg loss, 0.000010 rate, 3.135460 seconds, 382464 images
Loaded: 0.000074 seconds

 11953: 0.011296, 0.011899 avg loss, 0.000010 rate, 3.147467 seconds, 382496 images
Loaded: 0.000067 seconds

 11954: 0.007748, 0.011483 avg loss, 0.000010 rate, 3.138968 seconds, 382528 images
Loaded: 0.000043 seconds

 11955: 0.012709, 0.011606 avg loss, 0.000010 rate, 3.142545 seconds, 382560 images
Loaded: 0.000051 seconds

 11956: 0.008744, 0.011320 avg loss, 0.000010 rate, 3.142365 seconds, 382592 images
Loaded: 0.000043 seconds

 11957: 0.011090, 0.011297 avg loss, 0.000010 rate, 3.139540 seconds, 382624 images
Loaded: 0.000041 seconds

 11958: 0.015132, 0.011680 avg loss, 0.000010 rate, 3.151899 seconds, 382656 images
Loaded: 0.000068 seconds

 11959: 0.014476, 0.011960 avg loss, 0.000010 rate, 3.143687 seconds, 382688 images
Loaded: 0.000068 seconds

 11960: 0.012538, 0.012018 avg loss, 0.000010 rate, 3.150144 seconds, 382720 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.042592 seconds

 11961: 0.016780, 0.012494 avg loss, 0.000010 rate, 2.086771 seconds, 382752 images
Loaded: 0.000046 seconds

 11962: 0.012618, 0.012506 avg loss, 0.000010 rate, 2.174875 seconds, 382784 images
Loaded: 0.000043 seconds

 11963: 0.016101, 0.012866 avg loss, 0.000010 rate, 2.179757 seconds, 382816 images
Loaded: 0.000049 seconds

 11964: 0.010906, 0.012670 avg loss, 0.000010 rate, 2.178957 seconds, 382848 images
Loaded: 0.000067 seconds

 11965: 0.009633, 0.012366 avg loss, 0.000010 rate, 2.180642 seconds, 382880 images
Loaded: 0.000042 seconds

 11966: 0.015218, 0.012651 avg loss, 0.000010 rate, 2.170472 seconds, 382912 images
Loaded: 0.000041 seconds

 11967: 0.013705, 0.012757 avg loss, 0.000010 rate, 2.174548 seconds, 382944 images
Loaded: 0.000042 seconds

 11968: 0.012395, 0.012721 avg loss, 0.000010 rate, 2.175929 seconds, 382976 images
Loaded: 0.000034 seconds

 11969: 0.013538, 0.012802 avg loss, 0.000010 rate, 2.169688 seconds, 383008 images
Loaded: 0.000042 seconds

 11970: 0.015177, 0.013040 avg loss, 0.000010 rate, 2.171900 seconds, 383040 images
Resizing
608 x 608 
 try to allocate additional workspace_size = 108.69 MB 
 CUDA allocate done! 
Loaded: 0.000104 seconds

 11971: 0.015158, 0.013252 avg loss, 0.000010 rate, 4.935349 seconds, 383072 images
Loaded: 0.000072 seconds

 11972: 0.029524, 0.014879 avg loss, 0.000010 rate, 5.125213 seconds, 383104 images
Loaded: 0.000043 seconds

 11973: 0.011311, 0.014522 avg loss, 0.000010 rate, 5.124560 seconds, 383136 images
Loaded: 0.000062 seconds

 11974: 0.016504, 0.014720 avg loss, 0.000010 rate, 5.132992 seconds, 383168 images
Loaded: 0.000045 seconds

 11975: 0.007569, 0.014005 avg loss, 0.000010 rate, 5.137850 seconds, 383200 images
Loaded: 0.000040 seconds

 11976: 0.030341, 0.015639 avg loss, 0.000010 rate, 5.140753 seconds, 383232 images
Loaded: 0.000061 seconds

 11977: 0.013381, 0.015413 avg loss, 0.000010 rate, 5.131386 seconds, 383264 images
Loaded: 0.000047 seconds

 11978: 0.019122, 0.015784 avg loss, 0.000010 rate, 5.136320 seconds, 383296 images
Loaded: 0.000042 seconds

 11979: 0.015500, 0.015755 avg loss, 0.000010 rate, 5.138267 seconds, 383328 images
Loaded: 0.000058 seconds

 11980: 0.011939, 0.015374 avg loss, 0.000010 rate, 5.126741 seconds, 383360 images
Resizing
544 x 544 
 try to allocate additional workspace_size = 87.01 MB 
 CUDA allocate done! 
Loaded: 0.000072 seconds

 11981: 0.013272, 0.015164 avg loss, 0.000010 rate, 4.191016 seconds, 383392 images
Loaded: 0.000079 seconds

 11982: 0.014905, 0.015138 avg loss, 0.000010 rate, 4.331334 seconds, 383424 images
Loaded: 0.000045 seconds

 11983: 0.009641, 0.014588 avg loss, 0.000010 rate, 4.352576 seconds, 383456 images
Loaded: 0.000043 seconds

 11984: 0.013971, 0.014526 avg loss, 0.000010 rate, 4.337270 seconds, 383488 images
Loaded: 0.000042 seconds

 11985: 0.011357, 0.014209 avg loss, 0.000010 rate, 4.334991 seconds, 383520 images
Loaded: 0.000068 seconds

 11986: 0.008964, 0.013685 avg loss, 0.000010 rate, 4.336779 seconds, 383552 images
Loaded: 0.000042 seconds

 11987: 0.020524, 0.014369 avg loss, 0.000010 rate, 4.334672 seconds, 383584 images
Loaded: 0.000071 seconds

 11988: 0.010998, 0.014032 avg loss, 0.000010 rate, 4.337513 seconds, 383616 images
Loaded: 0.000045 seconds

 11989: 0.012945, 0.013923 avg loss, 0.000010 rate, 4.329368 seconds, 383648 images
Loaded: 0.000045 seconds

 11990: 0.019760, 0.014507 avg loss, 0.000010 rate, 4.333880 seconds, 383680 images
Resizing
384 x 384 
 try to allocate additional workspace_size = 52.43 MB 
 CUDA allocate done! 
Loaded: 0.000059 seconds

 11991: 0.018834, 0.014940 avg loss, 0.000010 rate, 2.085105 seconds, 383712 images
Loaded: 0.000055 seconds

 11992: 0.012876, 0.014733 avg loss, 0.000010 rate, 2.162809 seconds, 383744 images
Loaded: 0.000045 seconds

 11993: 0.011104, 0.014370 avg loss, 0.000010 rate, 2.164764 seconds, 383776 images
Loaded: 0.000059 seconds

 11994: 0.016126, 0.014546 avg loss, 0.000010 rate, 2.169024 seconds, 383808 images
Loaded: 0.000071 seconds

 11995: 0.010130, 0.014104 avg loss, 0.000010 rate, 2.167438 seconds, 383840 images
Loaded: 0.000048 seconds

 11996: 0.013662, 0.014060 avg loss, 0.000010 rate, 2.172126 seconds, 383872 images
Loaded: 0.000069 seconds

 11997: 0.016249, 0.014279 avg loss, 0.000010 rate, 2.171730 seconds, 383904 images
Loaded: 0.000045 seconds

 11998: 0.019076, 0.014759 avg loss, 0.000010 rate, 2.166644 seconds, 383936 images
Loaded: 0.000040 seconds

 11999: 0.008589, 0.014142 avg loss, 0.000010 rate, 2.171869 seconds, 383968 images
Loaded: 0.000040 seconds

 12000: 0.009018, 0.013629 avg loss, 0.000010 rate, 2.079949 seconds, 384000 images
Saving weights to backup/custom-train-yolo_12000.weights
Saving weights to backup/custom-train-yolo_last.weights
Saving weights to backup/custom-train-yolo_final.weights

In [13]:

!cp -r ./backup "/content/drive/MyDrive/darknet/backup/29_game_train_yolo_files"