Home Explore Help
Register Sign In
brain-assistant
/
yet-another-applied-llm-benchmark
5
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Tree: 4d88c30242
Branches Tags
yet-another-app...
/
tests
/
make_json.py

make_json.py 2.1 KB
History Raw

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576 
  1. from evaluator import *
    2. 

  2. 

  3. DESCRIPTION = "Test if the model can successfully convert unstructured data to JSON."
    4. 

  4. 

  5. TAGS = ['data']
    6. 

  6. 

  7. question = """
    8. 

  8. Create a JSON metadata for these models:
    9. 

  9. 

  10. Mistral-7B-v0.1
    11. 

  11. RedPajama-INCITE-7B-Base
    12. 

  12. RedPajama-INCITE-Base-3B-v1
    13. 

  13. falcon40b
    14. 

  14. falcon7b
    15. 

  15. gpt2-xl
    16. 

  16. llama-65b
    17. 

  17. llama-7b
    18. 

  18. neo-1.3
    19. 

  19. neo-2.7
    20. 

  20. neo-6
    21. 

  21. open_llama_3b_v2
    22. 

  22. open_llama_7b_v2
    23. 

  23. opt-1.3b
    24. 

  24. opt-6.7b
    25. 

  25. pythia-1.4
    26. 

  26. pythia-1.4-dedup
    27. 

  27. pythia-6.9
    28. 

  28. pythia-6.9-dedup
    29. 

  29. 

  30. With the format:
    31. 

  31. 

  32. {"Mistral-7B-v0.1": {"size": 7, dataset: "", "family": "Mistral"}, ...}
    33. 

  33. 

  34. where family is one of 
    35. 

  35. 

  36.     base = [
    37. 

  37.         'pythia',
    38. 

  38.         'llama',
    39. 

  39.         'Mistral',
    40. 

  40.         'gpt2',
    41. 

  41.         'opt',
    42. 

  42.         'RedPajama',
    43. 

  43.         'neo',
    44. 

  44.         'open_llama',
    45. 

  45.         'falcon'
    46. 

  46.     ]
    47. 

  47. 

  48. gpt2-xl is 1.5b parameters.
    49. 

  49. 

  50. """
    51. 

  51. 

  52. 

  53. TestMakeJson = question >> LLMRun() >> ExtractJSON() >> JSONSubsetEvaluator({
    54. 

  54.   "Mistral-7B-v0.1": {"size": 7, "dataset": "", "family": "Mistral"},
    55. 

  55.   "RedPajama-INCITE-7B-Base": {"size": 7, "dataset": "", "family": "RedPajama"},
    56. 

  56.   "RedPajama-INCITE-Base-3B-v1": {"size": 3, "dataset": "", "family": "RedPajama"},
    57. 

  57.   "falcon40b": {"size": 40, "dataset": "", "family": "falcon"},
    58. 

  58.   "falcon7b": {"size": 7, "dataset": "", "family": "falcon"},
    59. 

  59.   "gpt2-xl": {"size": 1.5, "dataset": "", "family": "gpt2"},
    60. 

  60.   "llama-65b": {"size": 65, "dataset": "", "family": "llama"},
    61. 

  61.   "llama-7b": {"size": 7, "dataset": "", "family": "llama"},
    62. 

  62.   "neo-1.3": {"size": 1.3, "dataset": "", "family": "neo"},
    63. 

  63.   "neo-2.7": {"size": 2.7, "dataset": "", "family": "neo"},
    64. 

  64.   "neo-6": {"size": 6, "dataset": "", "family": "neo"},
    65. 

  65.   "open_llama_3b_v2": {"size": 3, "dataset": "", "family": "open_llama"},
    66. 

  66.   "open_llama_7b_v2": {"size": 7, "dataset": "", "family": "open_llama"},
    67. 

  67.   "opt-1.3b": {"size": 1.3, "dataset": "", "family": "opt"},
    68. 

  68.   "opt-6.7b": {"size": 6.7, "dataset": "", "family": "opt"},
    69. 

  69.   "pythia-1.4": {"size": 1.4, "dataset": "", "family": "pythia"},
    70. 

  70.   "pythia-1.4-dedup": {"size": 1.4, "dataset": "", "family": "pythia"},
    71. 

  71.   "pythia-6.9": {"size": 6.9, "dataset": "", "family": "pythia"},
    72. 

  72.   "pythia-6.9-dedup": {"size": 6.9, "dataset": "", "family": "pythia"}
    73. 

  73. })
    74. 

  74. 

  75. if __name__ == "__main__":
    76. 

  76.     print(run_test(TestMakeJson))
    77.