Efficiently Breaking a Loop into "Chunks"-CodePudding

I have this code in R:

final_results <- list()

for (i in 1:339)

{

index_i = i
num_i = rnorm(1,1,1)
temp_i = data.frame(index_i,num_i)

final_results[[i]] <- temp_i
   
}

results_dataframe <- do.call(rbind.data.frame, final_results)

I would like to break this code into "chunks" of 100. For example:

# break into chunks
# chunk 1

 final_results <- list()
    
    for (i in 1:100)
    
    {
    
    index_i = i
    num_i = rnorm(1,1,1)
    temp_i = data.frame(index_i,num_i)
    
    final_results[[i]] <- temp_i
       
    }
    
    results_dataframe_1 <- do.call(rbind.data.frame, final_results)

# chunk 2
final_results <- list()
    
    for (i in 101:201)
    
    {
    
    index_i = i
    num_i = rnorm(1,1,1)
    temp_i = data.frame(index_i,num_i)
    
    final_results[[i]] <- temp_i
       
    }
    
    results_dataframe_2 <- do.call(rbind.data.frame, final_results)

# chunk 3

final_results <- list()
    
    for (i in 202:302)
    
    {
    
    index_i = i
    num_i = rnorm(1,1,1)
    temp_i = data.frame(index_i,num_i)
    
    final_results[[i]] <- temp_i
       
    }
    
    results_dataframe_3 <- do.call(rbind.data.frame, final_results)

# chunk 4

final_results <- list()
    
    for (i in 303:339)
    
    {
    
    index_i = i
    num_i = rnorm(1,1,1)
    temp_i = data.frame(index_i,num_i)
    
    final_results[[i]] <- temp_i
       
    }
    
    results_dataframe_4 <- do.call(rbind.data.frame, final_results)

# bind everything together

final_result = rbind(results_dataframe_1, results_dataframe_2, results_dataframe_3, results_dataframe_4)

I am looking for a more efficient way to do this. For example, suppose the index went from 1:887 - I would have to copy and modify this code several times.

I thought of the following way to start this problem - I can "break the index" into some arbitrary number of "chunks":

d = 1:339
res<-as.integer(quantile(d, probs=seq(0.1,1, by=0.1)))

   [1]  34  68 102 136 170 203 237 271 305 339

But is there a way to take these quantiles and efficiently run the loop by these "chunks"?

Thank you!

CodePudding user response：

Based on the for loop code, we may need a group by option i.e. create a grouping index with gl that increments 1 every 100 values, then apply rnorm and later rbind the list elements

lst1 <- tapply(1:339, as.integer(gl(339, 100, 339)), 
     FUN = function(x) rnorm(length(x), 1, 1))
out <- do.call(rbind, Map(\(x, y) data.frame(grp = x, val = y), 
   names(lst1), lst1))

CodePudding user response：

If you want to be able to keep the chunks so that they can be run in a loop (e.g. if the computation is slow and you want to be able to check progress), then one option to get fixed chunk sizes would be:

n <- 339
chunk_size <- 100

chunks <- split(seq(n), seq(n) %/% chunk_size)

This results in

chunks
#> $`0`
#>  [1]  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#> [26] 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
#> [51] 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
#> [76] 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
#> 
#> $`1`
#>   [1] 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
#>  [19] 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
#>  [37] 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
#>  [55] 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
#>  [73] 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
#>  [91] 190 191 192 193 194 195 196 197 198 199
#> 
#> $`2`
#>   [1] 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
#>  [19] 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
#>  [37] 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
#>  [55] 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
#>  [73] 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
#>  [91] 290 291 292 293 294 295 296 297 298 299
#> 
#> $`3`
#>  [1] 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
#> [20] 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
#> [39] 338 339

This means that you can run a loop that only iterates 100 at a time, giving

all_results <- list()

for(i in seq_along(chunks)) {
  all_results[[i]] <- do.call(rbind, lapply(chunks[[i]], function(i) {
    data.frame(index_i = i, num_i = rnorm(1, 1, 1))
  }))
}

This gives us a list of four data frames, which you could then bind at the end.

str(all_results)
#> List of 4
#>  $ :'data.frame':    99 obs. of  2 variables:
#>   ..$ index_i: int [1:99] 1 2 3 4 5 6 7 8 9 10 ...
#>   ..$ num_i  : num [1:99] 0.5198 -0.0456 0.7916 0.4286 -0.2205 ...
#>  $ :'data.frame':    100 obs. of  2 variables:
#>   ..$ index_i: int [1:100] 100 101 102 103 104 105 106 107 108 109 ...
#>   ..$ num_i  : num [1:100] -0.27 1.767 -0.979 2.163 0.911 ...
#>  $ :'data.frame':    100 obs. of  2 variables:
#>   ..$ index_i: int [1:100] 200 201 202 203 204 205 206 207 208 209 ...
#>   ..$ num_i  : num [1:100] -0.799 2.709 1.516 1.719 0.878 ...
#>  $ :'data.frame':    40 obs. of  2 variables:
#>   ..$ index_i: int [1:40] 300 301 302 303 304 305 306 307 308 309 ...
#>   ..$ num_i  : num [1:40] -0.498 0.742 1.92 1.991 2.173 ...

^{Created on 2022-07-14 by the reprex package (v2.0.1)}