merge_into_16.py 2.1 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455
  1. from evaluator import *
  2. DESCRIPTION = "Test if the model can write a Python script that merges a list of file paths into 16 files of approximately equal size."
  3. TAGS = ['code', 'python']
  4. question_easy = """
  5. I have a list of several thousand file paths in /tmp/files. Write a python script that merges these all together and splits the content into 16 files called /tmp/merged_%d for i in 0..15 where each is approximately the same size. To do this, sort files by size (largest first) and add files in order always to the smallest one. "/tmp/files" is a list of file paths, one per line.
  6. """
  7. question = """
  8. I have a list of several thousand file paths in /tmp/files. Write a python script that merges the content int 16 files called /tmp/merged_%d for i in 0..15 where each is approximately the same size. "/tmp/files" is a list of file paths, one per line.
  9. """
  10. def setup_fn():
  11. import random
  12. import os
  13. os.mkdir("/tmp/a")
  14. with open("/tmp/files", "w") as f:
  15. for i in range(1000):
  16. f.write("/tmp/a/file_%d\n" % i)
  17. with open("/tmp/a/file_%d" % i, "w") as f2:
  18. if i < 5:
  19. f2.write("x" * random.randint(10000, 30000))
  20. else:
  21. f2.write("x" * random.randint(1, 1000))
  22. def test():
  23. import os
  24. import numpy as np
  25. sz = []
  26. for i in range(16):
  27. sz.append(len(open("/tmp/merged_%d" % i).read()))
  28. size_before = sum(os.path.getsize("/tmp/a/file_%d"%i) for i in range(1000))
  29. size_after = sum(sz)
  30. # I ddn't say exactly how to merge, adding newlines is okay
  31. if abs(size_before-size_after) > 2000:
  32. return False
  33. # easy to hit with proper merging, hard otherwise
  34. # 10 trials suggest the range is usualy [10, 40]
  35. return max(sz) - min(sz) < 200
  36. TestMake16FilesEasy = Setup(setup_fn) >> question_easy >> LLMRun() >> ExtractCode(keep_main=True) >> Echo() >> PythonRun() >> PyEvaluator(test)
  37. TestMake16Files = Setup(setup_fn) >> question >> LLMRun() >> ExtractCode(keep_main=True) >> Echo() >> PythonRun() >> PyEvaluator(test)
  38. if __name__ == "__main__":
  39. print(run_test(TestMake16FilesEasy))