init repo

.gitignore

@@ -0,0 +1,1225 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+.idea
+*.iml
+out
+gen
+### CVS template
+/CVS/*
+**/CVS/*
+.cvsignore
+*/.cvsignore
+
+### C++ template
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+### CUDA template
+*.i
+*.ii
+*.gpu
+*.ptx
+*.cubin
+*.fatbin
+
+### SVN template
+.svn/
+
+### Images template
+# JPEG
+*.jpe
+*.jif
+*.jfif
+*.jfi
+
+# JPEG 2000
+*.jp2
+*.j2k
+*.jpf
+*.jpx
+*.jpm
+*.mj2
+
+# JPEG XR
+*.jxr
+*.hdp
+*.wdp
+
+# Graphics Interchange Format
+*.gif
+
+# RAW
+*.raw
+
+# Web P
+*.webp
+
+# Portable Network Graphics
+#*.png
+
+# Animated Portable Network Graphics
+*.apng
+
+# Multiple-image Network Graphics
+*.mng
+
+# Tagged Image File Format
+*.tiff
+*.tif
+
+# Scalable Vector Graphics
+*.svg
+*.svgz
+
+# Portable Document Format
+*.pdf
+
+# X BitMap
+*.xbm
+
+# BMP
+*.bmp
+*.dib
+
+# ICO
+*.ico
+
+# 3D Images
+*.3dm
+*.max
+
+### Eclipse template
+.metadata
+bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.recommenders
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# PyDev specific (Python IDE for Eclipse)
+*.pydevproject
+
+# CDT-specific (C/C++ Development Tooling)
+.cproject
+
+# CDT- autotools
+.autotools
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific (PHP Development Tools)
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# Tern plugin
+.tern-project
+
+# TeXlipse plugin
+.texlipse
+
+# STS (Spring Tool Suite)
+.springBeans
+
+# Code Recommenders
+.recommenders/
+
+# Annotation Processing
+.apt_generated/
+.apt_generated_test/
+
+# Scala IDE specific (Scala & Java development for Eclipse)
+.cache-main
+.scala_dependencies
+.worksheet
+
+# Uncomment this line if you wish to ignore the project description file.
+# Typically, this file would be tracked if it contains build/dependency configurations:
+#.project
+
+### Diff template
+*.patch
+*.diff
+
+### macOS template
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### CMake template
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
+
+### Linux template
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### MicrosoftOffice template
+*.tmp
+
+# Word temporary
+~$*.doc*
+
+# Word Auto Backup File
+Backup of *.doc*
+
+# Excel temporary
+~$*.xls*
+
+# Excel Backup File
+*.xlk
+
+# PowerPoint temporary
+~$*.ppt*
+
+# Visio autosave temporary files
+*.~vsd*
+
+### VisualStudio template
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+### Vim template
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### Backup template
+*.bak
+*.gho
+*.ori
+*.orig
+*.tmp
+
+### SublimeText template
+# Cache files for Sublime Text
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+
+# Workspace files are user-specific
+*.sublime-workspace
+
+# Project files should be checked into the repository, unless a significant
+# proportion of contributors will probably not be using Sublime Text
+# *.sublime-project
+
+# SFTP configuration file
+sftp-config.json
+sftp-config-alt*.json
+
+# Package control specific files
+Package Control.last-run
+Package Control.ca-list
+Package Control.ca-bundle
+Package Control.system-ca-bundle
+Package Control.cache/
+Package Control.ca-certs/
+Package Control.merged-ca-bundle
+Package Control.user-ca-bundle
+oscrypto-ca-bundle.crt
+bh_unicode_properties.cache
+
+# Sublime-github package stores a github token in this file
+# https://packagecontrol.io/packages/sublime-github
+GitHub.sublime-settings
+
+### C template
+# Prerequisites
+*.d
+
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Linker output
+*.ilk
+*.map
+*.exp
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Debug files
+*.dSYM/
+*.su
+*.idb
+*.pdb
+
+# Kernel Module Compile Results
+*.mod*
+*.cmd
+.tmp_versions/
+modules.order
+Module.symvers
+Mkfile.old
+dkms.conf
+
+### VirtualEnv template
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+pip-selfcheck.json
+
+### LibreOffice template
+# LibreOffice locks
+.~lock.*#
+
+### Xilinx template
+# gitignore template for Xilinx Vivado Design Suite
+# website: https://www.xilinx.com/support/download.html
+
+# [home]
+*.jou
+*.log
+*.debug
+*.str
+*.tmp
+*.os
+*.js
+*.pb
+*.dcp
+*.hwdef
+*.vds
+*.veo
+*.wdf
+*.vdi
+*.dmp
+*.rpx
+*.rpt
+*_stub.v
+*_stub.vhdl
+*_funcsim.v
+*_funcsim.vhdl
+.project
+
+# [dir]
+*.cache
+.metadata
+*.data
+*.ipdefs
+.Xil
+*.sdk
+*.hw
+*.ip_user_files
+
+### IP synth
+*_synth_*
+
+.jobs
+
+### project synth
+*/*.runs/synth*/*.xml
+*/*.runs/synth*/*.txt
+*/*.runs/synth*/*.sh
+*/*.runs/synth*/*.tcl
+*/*.runs/synth*/*.bat
+*/*.runs/synth*/*.xdc
+!*/*.runs/synth*/*utilization*.rpt
+
+*.runs/synth*/*.xml
+*.runs/synth*/*.txt
+*.runs/synth*/*.sh
+*.runs/synth*/*.tcl
+*.runs/synth*/*.bat
+*.runs/synth*/*.xdc
+!*.runs/synth*/*utilization*.rpt
+
+### project impl
+*/*.runs/impl*/*.xml
+*/*.runs/impl*/*.html
+*/*.runs/impl*/*.txt
+*/*.runs/impl*/*.sh
+*/*.runs/impl*/*.tcl
+*/*.runs/impl*/*.bat
+!*/*.runs/impl*/*utilization*.rpt
+
+*.runs/impl*/*.xml
+*.runs/impl*/*.html
+*.runs/impl*/*.txt
+*.runs/impl*/*.sh
+*.runs/impl*/*.tcl
+*.runs/impl*/*.bat
+!*.runs/impl*/*utilization*.rpt
+
+### block design
+*/*/bd/*/hdl
+*/*/*/bd/*/hdl
+
+*/*/bd/*/*.xdc
+*/*/*/bd/*/*.xdc
+
+*/*/bd/*/ip/*/*.xdc
+*/*/*/bd/*/ip/*/*.xdc
+
+*/*/bd/*/ip/*/*/
+*/*/*/bd/*/ip/*/*/
+
+*/*/bd/*/ip/*/*.vhd
+*/*/*/bd/*/ip/*/*.vhd
+
+*/*/bd/*/ip/*/*.xml
+*/*/*/bd/*/ip/*/*.xml
+
+*.c
+*.cpp
+*.h
+*.vho
+*/*/bd/*/ip/*/*.tcl
+*/*/*/bd/*/ip/*/*.tcl
+hw_handoff
+ipshared
+
+### Lua template
+# Compiled Lua sources
+luac.out
+
+# luarocks build files
+*.src.rock
+*.zip
+
+# Object files
+*.o
+*.os
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+*.def
+*.exp
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+
+### Vagrant template
+# General
+.vagrant/
+
+# Log files (if you are creating logs in debug mode, uncomment this)
+# *.log
+
+### PuTTY template
+# Private key
+*.ppk
+
+### Patch template
+*.orig
+*.rej
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+/dist
+wheelhouse/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+/env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### Windows template
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+### Xcode template
+# Xcode
+#
+# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
+
+## User settings
+xcuserdata/
+
+## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
+*.xcscmblueprint
+*.xccheckout
+
+## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
+build/
+DerivedData/
+*.moved-aside
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+
+## Gcc Patch
+/*.gcno
+
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+/test_*
+.python-version
+/docs/build
+/public
+/docs/source/**/*.puml.svg
+/docs/source/**/*.puml.png
+/docs/source/**/*.gv.svg
+/docs/source/**/*.gv.png
+/docs/source/**/*.py.txt
+/docs/source/**/*.py.err
+/docs/source/**/*.py.exitcode
+/docs/source/**/*.sh.txt
+/docs/source/**/*.sh.err
+/docs/source/**/*.sh.exitcode
+/docs/source/**/*.dat.*
+!/docs/source/**/*.plot.py.svg
+!/docs/source/_static/**/*
+/docs/source/**/*.result.ipynb
+/cartpole_dqn_*
+/llmriddles/**/*.json
+/llmriddles/**/*.yaml
+/llmriddles/**/*.yml
+/extra
+/resource/yolort/*
+!/resource/yolort/Makefile
+!/test/testfile/**/*
+!/zoo/testfile/**/*
+/test/testfile/dataset
+/zoo/testfile/dataset
+/.train
+*.pt
+/runs
+/YOLOv8

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README_zh.md

@@ -0,0 +1,95 @@
+# LLM Riddles
+
+<div align="center">
+	<br>
+	<a href="https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/banner.svg">
+		<img src="https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/banner.svg" width="1000" height="200" alt="Click to see the source">
+	</a>
+	<br>
+</div>
+
+[English](https://github.com/opendilab/LLMRiddles/blob/main/README.md) | 简体中文
+
+## :thinking: 什么是LLM Riddles
+欢迎来到 LLM Riddles！这是一个与语言模型斗智斗勇的游戏。在游戏中，你需要构造与语言模型交互的问题，来得到符合要求的答案。在这个过程中，你可以开动脑筋，用你想到的所有方式，让模型输出答案要求的结果。
+
+## :space_invader: 如何试玩
+我们提供了在线版本以供玩家直接访问试玩:
+- [ChatGPT + 英文(需配置api key)](https://huggingface.co/spaces/OpenDILabCommunity/LLMRiddlesChatGPTEN)
+- [ChatGPT + 中文(需配置api key)](https://huggingface.co/spaces/OpenDILabCommunity/LLMRiddlesChatGPTCN)
+- [Mistral + 英文(已预设api key)](https://d9b451a97791dd8ef3.gradio.live)
+- [ChatGPT + 中文(已预设api key)](http://llmriddles.opendilab.net/)
+
+本地部署可以通过以下方式：
+## 安装
+### ChatGPT 或 ChatGLM API
+```shell
+pip3 install -r requirements.txt
+```
+### Mistral-7B-Instruct-v0.1 本地推理
+```shell
+pip3 install -r requirements-dev.txt
+```
+## 启动
+### ChatGPT + 中文
+```shell
+QUESTION_LANG=cn QUESTION_LLM='chatgpt' QUESTION_LLM_KEY=<your API key> python3 -u app.py
+```
+### ChatGPT + 英文
+```shell
+QUESTION_LANG=en QUESTION_LLM='chatgpt' QUESTION_LLM_KEY=<your API key> python3 -u app.py
+```
+### ChatGLM + 中文
+```shell
+QUESTION_LANG=cn QUESTION_LLM='chatglm' QUESTION_LLM_KEY=<your API key> python3 -u app.py
+```
+### ChatGLM + 英文
+```shell
+QUESTION_LANG=en QUESTION_LLM='chatglm' QUESTION_LLM_KEY=<your API key> python3 -u app.py
+```
+### Mistral-7B-Instruct-v0.1 + 英文
+```shell
+QUESTION_LANG=en QUESTION_LLM='mistral-7b' python3 -u app.py
+```
+## :technologist: 为什么制作这个游戏
+
+我们的目标是通过这一游戏，让参与者深入领略到提示工程（prompt engineering）和自然语言处理的令人着迷之处。这个过程将向玩家们展示，如何巧妙地构建提示词（prompts），以及如何运用它们来引发人工智能系统的惊人反应，同时也帮助他们更好地理解深度学习和自然语言处理技术的不可思议之处。
+
+## :raising_hand: 如何提交设计好的关卡
+如果有好玩的问题或想法，欢迎玩家提交自己的创意，可以
+[发起 Pull Request](https://github.com/opendilab/LLMRiddles/compare) 向我们提交， 我们会在审核通过后收录至关卡中。
+问题的设计格式应包含以下几点：
+- Pull Request标题，示例：feature(username): 章节X-关卡设计
+- 希望被提及的ID
+- 对应章节问题文件的修改
+- \__init__.py的修改
+
+完整示例请参考：[提交属于自己的关卡设计](https://github.com/opendilab/LLMRiddles/pull/6)
+
+## :writing_hand: 未来计划
+
+- [x] 支持自定义关卡
+- [x] 在线试玩链接
+- [x] Hugging Face Space 链接
+- [x] 支持Mistral-7B（英文）
+- [x] 支持ChatGLM（中文和英文）
+- [ ] 支持Baichuan2-7B（中文）
+- [ ] 支持LLaMA2-7B（英文）
+- [ ] LLM 推理速度优化
+
+## :speech_balloon: 反馈问题 & 提出建议
+- 在 GitHub 上[发起 Issue](https://github.com/opendilab/CodeMorpheus/issues/new/choose)
+- 通过邮件与我们联系 ([email protected])
+- 在OpenDILab的群组中加入讨论(通过 WeChat: ding314assist 添加小助手微信)
+<img src=https://github.com/opendilab/LLMRiddles/blob/main/llmriddles/assets/wechat.jpeg width=35% />
+
+## :star2: Special Thanks
+- 感谢 [Haoqiang Fan](https://www.zhihu.com/people/haoqiang-fan) 的原始创意和题目，为本项目的开发和扩展提供了灵感与动力。
+- 感谢 [HuggingFace](https://huggingface.co) 对游戏的支持与协助。
+- 感谢 [ChatGLM](https://chatglm.cn/) 对游戏的支持与协助，特别是供在线预览版使用的足量 tokens。
+- 感谢 [LLM Riddles contributors](https://github.com/opendilab/LLMRiddles/graphs/contributors) 的实现与支持。
+
+## :label: License
+All code within this repository is under [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0).
+
+<p align="right">(<a href="#top">back to top</a>)</p>

app.py

@@ -0,0 +1,266 @@
+import logging
+import os
+import uuid
+
+import gradio as gr
+
+from llmriddles.questions import QuestionExecutor
+from llmriddles.questions import list_ordered_questions
+
+_QUESTION_SESSIONS = {}
+count = 0
+_QUESTIONS = list_ordered_questions()
+_LANG = os.environ.get('QUESTION_LANG', 'cn')
+assert _LANG in ['cn', 'en'], _LANG
+_LLM = os.environ.get('QUESTION_LLM', 'chatgpt')
+assert _LLM in ['chatgpt', 'chatglm', 'mistral-7b'], _LLM
+_LLM_KEY = os.environ.get('QUESTION_LLM_KEY', None)
+_DEBUG = os.environ.get('DEBUG', 'false').lower() == 'true'
+
+if _DEBUG:
+    logging.getLogger().setLevel(logging.INFO)
+else:
+    logging.getLogger().setLevel(logging.WARNING)
+if _LANG == "cn":
+    title = "完蛋！我被 LLM 拿捏了"
+    requirement_ph = """
+    <h2 style="color: #6d28d9;"> 欢迎来到 LLM Riddles! </h2>
+    <h4> 你将通过本游戏对大语言模型产生更深刻的理解。在本游戏中，你需要构造一个提给语言大模型的问题，使得它回复的答案符合题目要求。点击<i>\"下一题\"</i> 即可开始游戏。</h4>
+    """
+    requirement_label = "游戏须知/说明"
+    question_ph = "你对大语言模型的提问（例如：请你输出1+1=3）"
+    question_label = "玩家提问栏"
+    answer_ph = "大语言模型的回答"
+    answer_label = "大语言模型回答栏"
+    submit_label = "提交"
+    next_label = "下一题"
+    api_ph = "你个人的大语言模型 API Key (例如：ChatGPT)"
+    api_label = "API key"
+    predict_label = "结果正确性"
+    explanation_label = "结果详细解释"
+    game_cleared_label = "<h2 style='color: #6d28d9;'>祝贺！你已成功通关！</h2>"
+    correct_label = "正确"
+    wrong_label = "错误"
+    api_error_info = "请在提交问题之前先输入你的 API Key"
+    try_again_label = "再玩一次"
+    select_label = "选择关卡（投机取巧需谨慎）"
+    title_markdown = """
+    <div align="center">
+        <img src="https://raw.githubusercontent.com/opendilab/LLMRiddles/main/llmriddles/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
+    </div>
+    <h2 style="text-align: center; color: black;"><a href="https://github.com/OpenDILab/LLMRiddles"> 🎭LLM Riddles：完蛋！我被 LLM 拿捏了</a></h2>
+    <strong><h5 align="center"> 更多不同语言模型的在线试玩 demo 可以访问 GitHub<a href="https://github.com/OpenDILab/LLMRiddles">源代码仓库</a>获取<h5></strong>
+    <h5 align="center"> 如果你喜欢这个项目，请给我们在 GitHub 点个 star ✨ <a href="https://github.com/OpenDILab/LLMRiddles"> 代码仓库传送门 </a> 。我们将会持续保持更新。再次感谢游戏<a href="https://www.zhihu.com/people/haoqiang-fan"> 原作者 </a>的奇思妙想！  </h5>
+    <strong><h5 align="center">注意：算法模型的输出可能包含一定的随机性。相关结果不代表任何开发者和相关 AI 服务的态度和意见。本项目开发者不对生成结果作任何保证，仅供娱乐。<h5></strong>
+    """
+    tos_markdown = """
+    ### 使用条款
+    玩家使用本服务须同意以下条款：
+    该服务是一项探索性研究预览版，仅供非商业用途。它仅提供有限的安全措施，并可能生成令人反感的内容。不得将其用于任何非法、有害、暴力、种族主义等目的。该服务可能会收集玩家对话数据以供未来研究之用。
+    如果您的游玩体验有不佳之处，请发送邮件至 [email protected] ！ 我们将删除相关信息，并不断改进这个项目。
+    为了获得最佳体验，请使用台式电脑进行此预览版游戏，因为移动设备可能会影响可视化效果。
+    **版权所有 2023 OpenDILab。**
+    """
+elif _LANG == "en":
+    title = "LLM Riddles: Oops! Rolling in LLM."
+    requirement_ph = """
+    <h2 style="color: #6d28d9;">Welcome to LLM Riddles! </h2>
+    <h4> In this game, you'll gain a deeper understanding of language models. Your challenge is to create a question to ask a language model in a way that the answer it provides meets specific criteria. Click <i>\'Next\'</i> to Start</h4>
+    """
+    requirement_label = "Game Requirements"
+    question_ph = "Your Question for LLM (e.g. Please print 1+1=3)"
+    question_label = "Question"
+    answer_ph = "Answer From LLM"
+    answer_label = "Answer"
+    submit_label = "Submit"
+    next_label = "Next"
+    api_ph = "Your API Key (e.g. ChatGPT)"
+    api_label = "API key"
+    predict_label = "Correctness"
+    explanation_label = "Explanation"
+    game_cleared_label = "<h2 style='color: #6d28d9;'>Congratulations!</h2>"
+    correct_label = "Correct"
+    wrong_label = "Wrong"
+    api_error_info = "Please Enter API Key Before Submitting Question."
+    try_again_label = "Try Again"
+    select_label = "Select level"
+    title_markdown = """
+    <div align="center">
+        <img src="https://raw.githubusercontent.com/opendilab/LLMRiddles/main/llmriddles/assets/banner.svg" width="80%" height="20%" alt="Banner Image">
+    </div>
+    <h2 style="text-align: center; color: black;"><a href="https://github.com/OpenDILab/LLMRiddles"> 🎭LLM Riddles: Oops! Rolling in LLM.</a></h2>
+    <h5 align="center"> If you like our project, please give us a star ✨ on GitHub for latest update <a href="https://github.com/OpenDILab/LLMRiddles"> (Code Link) </a>. Thanks for the interesting idea of the original game <a href="https://www.zhihu.com/people/haoqiang-fan"> author </a>.  </h5>
+    <strong><h5 align="center">Notice: The output is generated by algorithm scheme and may involve some randomness. It does not represent the attitudes and opinions of any developers and AI services in this project. We do not make any guarantees about the generated content.<h5></strong>
+    """
+    tos_markdown = """
+    ### Terms of use
+    By using this service, players are required to agree to the following terms:
+    The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
+    Please send email to [email protected] if you get any inappropriate answer! We will delete those and keep improving our moderator.
+    For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+    **Copyright 2023 OpenDILab.**
+    """
+else:
+    raise KeyError("invalid _LANG: {}".format(_LANG))
+
+
+def _need_api_key():
+    return (_LLM == 'chatgpt' or _LLM == 'chatglm') and _LLM_KEY is None
+
+
+def _get_api_key_cfgs(api_key):
+    if _LLM == 'chatgpt':
+        return {'api_key': api_key}
+    elif _LLM == 'chatglm':
+        return {'api_key': api_key}
+    else:
+        return {}
+
+
+if __name__ == '__main__':
+    with gr.Blocks(title=title, theme='ParityError/Interstellar') as demo:
+        gr.Markdown(title_markdown)
+
+        with gr.Row():
+            gr_requirement = gr.HTML(value=requirement_ph, label=requirement_label)
+        with gr.Row():
+            with gr.Column():
+                gr_question = gr.TextArea(placeholder=question_ph, label=question_label)
+                gr_api_key = gr.Text(placeholder=api_ph, label=api_label, type='password', visible=_need_api_key())
+                with gr.Row():
+                    gr_submit = gr.Button(submit_label, interactive=False)
+                    gr_next = gr.Button(next_label)
+                with gr.Row():
+                    gr_select = gr.Radio(
+                        choices=[(QuestionExecutor(q, _LANG).question_name, i) for i, q in enumerate(_QUESTIONS)],
+                        label=select_label
+                    )
+
+            with gr.Column():
+                gr_uuid = gr.Text(value='', visible=False)
+                gr_predict = gr.Label(label=predict_label)
+                gr_answer = gr.TextArea(label=answer_label, lines=3)
+                gr_explanation = gr.TextArea(label=explanation_label, lines=1)
+        gr.Markdown(tos_markdown)
+
+        def _postprocess_question_text(question_text):
+            if _LANG == 'cn':
+                idx = question_text.find('，')
+                question_title = question_text[:idx]
+                former, latter = question_title.split('（')
+                question_title = former + '：' + latter[:-1]
+                question_text = f"<h2 style='color: #6d28d9;'>{question_title}</h2><h4>{question_text[idx+1:]}</h4>"
+            elif _LANG == 'en':
+                idx = question_text.find(',')
+                question_text = f"<h2 style='color: #6d28d9;'>{question_text[:idx]}</h2><h4>{question_text[idx+1:]}</h4>"
+            return question_text
+
+
+        def _radio_select(uuid_, select_qid):
+            global count
+            if not uuid_:
+                uuid_ = str(uuid.uuid4())
+                count += 1
+                logging.info(f'Player {count} starts the game now')
+            global _QUESTION_SESSIONS
+            if uuid_ not in _QUESTION_SESSIONS:
+                _QUESTION_SESSIONS[uuid_] = set(), select_qid
+            else:
+                _exists, _ = _QUESTION_SESSIONS[uuid_]
+                _QUESTION_SESSIONS[uuid_] = _exists, select_qid
+
+            executor = QuestionExecutor(_QUESTIONS[select_qid], _LANG)
+            question_text = _postprocess_question_text(executor.question_text)
+            return question_text, '', '', {}, '', \
+                gr.Button(submit_label, interactive=True), \
+                gr.Button(next_label, interactive=False), \
+                uuid_
+
+        gr_select.select(
+            _radio_select,
+            inputs=[gr_uuid, gr_select],
+            outputs=[
+                gr_requirement, gr_question, gr_answer,
+                gr_predict, gr_explanation, gr_submit, gr_next, gr_uuid,
+            ],
+        )
+
+
+        def _next_question(uuid_):
+            global count
+            if not uuid_:
+                uuid_ = str(uuid.uuid4())
+                count += 1
+                logging.info(f'Player {count} starts the game now')
+            global _QUESTION_SESSIONS
+            if uuid_ in _QUESTION_SESSIONS:
+                _exists, _qid = _QUESTION_SESSIONS[uuid_]
+            else:
+                _exists, _qid = set(), -1
+            _qid += 1
+            _QUESTION_SESSIONS[uuid_] = _exists, _qid
+
+            if _qid >= len(_QUESTIONS):
+                del _QUESTION_SESSIONS[uuid_]
+                logging.info(f'Player {count} has passed the game now')
+                return game_cleared_label, '', '', {}, '', \
+                    gr.Button(submit_label, interactive=False), \
+                    gr.Button(try_again_label, interactive=True), \
+                    '', \
+                    gr.Radio(
+                        choices=[(QuestionExecutor(q, _LANG).question_name, i) for i, q in enumerate(_QUESTIONS)],
+                        label=select_label
+                    )
+            else:
+                executor = QuestionExecutor(_QUESTIONS[_qid], _LANG)
+                question_text = _postprocess_question_text(executor.question_text)
+                return question_text, '', '', {}, '', \
+                    gr.Button(submit_label, interactive=True), \
+                    gr.Button(next_label, interactive=False), \
+                    uuid_, \
+                    gr.Radio(
+                        choices=[(QuestionExecutor(q, _LANG).question_name, i) for i, q in enumerate(_QUESTIONS)],
+                        value=_qid,
+                        label=select_label,
+                    )
+
+
+        gr_next.click(
+            fn=_next_question,
+            inputs=[gr_uuid],
+            outputs=[
+                gr_requirement, gr_question, gr_answer,
+                gr_predict, gr_explanation, gr_submit, gr_next,
+                gr_uuid, gr_select,
+            ],
+        )
+
+
+        def _submit_answer(qs_text: str, api_key: str, uuid_: str):
+            global _QUESTION_SESSIONS
+            if _need_api_key() and not api_key:
+                raise gr.Error(api_error_info)
+
+            _exists, _qid = _QUESTION_SESSIONS[uuid_]
+            executor = QuestionExecutor(
+                _QUESTIONS[_qid], _LANG,
+                llm=_LLM, llm_cfgs=_get_api_key_cfgs(api_key) if _need_api_key() else {'api_key': _LLM_KEY}
+            )
+            answer_text, correctness, explanation = executor.check(qs_text)
+            labels = {correct_label: 1.0} if correctness else {wrong_label: 1.0}
+            if correctness:
+                _QUESTION_SESSIONS[uuid_] = (_exists | {_qid}), _qid
+                return answer_text, labels, explanation, gr.Button(next_label, interactive=True), uuid_
+            else:
+                return answer_text, labels, explanation, gr.Button(next_label, interactive=False), uuid_
+
+
+        gr_submit.click(
+            _submit_answer,
+            inputs=[gr_question, gr_api_key, gr_uuid],
+            outputs=[gr_answer, gr_predict, gr_explanation, gr_next, gr_uuid],
+        )
+
+    concurrency = int(os.environ.get('CONCURRENCY', os.cpu_count()))
+    favicon_path = os.path.join(os.path.dirname(__file__), 'llmriddles', 'assets', 'avatar.png')
+    demo.queue().launch(max_threads=concurrency, favicon_path=favicon_path, share=True)

cloc.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# This scripts counts the lines of code and comments in all source files
+# and prints the results to the command line. It uses the commandline tool
+# "cloc". You can either pass --loc, --comments or --percentage to show the
+# respective values only.
+# Some parts below need to be adapted to your project!
+
+# Get the location of this script.
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+# Run cloc - this counts code lines, blank lines and comment lines
+# for the specified languages. You will need to change this accordingly.
+# For C++, you could use "C++,C/C++ Header" for example.
+# We are only interested in the summary, therefore the tail -1
+SUMMARY="$(cloc "${SCRIPT_DIR}" --include-lang="Python" --md | tail -1)"
+
+# The $SUMMARY is one line of a markdown table and looks like this:
+# SUM:|101|3123|2238|10783
+# We use the following command to split it into an array.
+IFS='|' read -r -a TOKENS <<<"$SUMMARY"
+
+# Store the individual tokens for better readability.
+NUMBER_OF_FILES=${TOKENS[1]}
+COMMENT_LINES=${TOKENS[3]}
+LINES_OF_CODE=${TOKENS[4]}
+
+# To make the estimate of commented lines more accurate, we have to
+# subtract any copyright header which is included in each file.
+# For Fly-Pie, this header has the length of five lines.
+# All dumb comments like those /////////// or those // ------------
+# are also subtracted. As cloc does not count inline comments,
+# the overall estimate should be rather conservative.
+# Change the lines below according to your project.
+# DUMB_COMMENTS="$(grep -r -E '//////|// -----' "${SCRIPT_DIR}" | wc -l)"
+# COMMENT_LINES=$(($COMMENT_LINES - 5 * $NUMBER_OF_FILES - $DUMB_COMMENTS))
+
+# Print all results if no arguments are given.
+if [[ $# -eq 0 ]]; then
+  awk -v a=$LINES_OF_CODE \
+    'BEGIN {printf "Lines of source code: %6.1fk\n", a/1000}'
+  awk -v a=$COMMENT_LINES \
+    'BEGIN {printf "Lines of comments:    %6.1fk\n", a/1000}'
+  awk -v a=$COMMENT_LINES -v b=$LINES_OF_CODE \
+    'BEGIN {printf "Comment Percentage:   %6.1f%\n", 100*a/b}'
+  exit 0
+fi
+
+# Show lines of code if --loc is given.
+if [[ $* == *--loc* ]]; then
+  awk -v a=$LINES_OF_CODE \
+    'BEGIN {printf "%.1fk\n", a/1000}'
+fi
+
+# Show lines of comments if --comments is given.
+if [[ $* == *--comments* ]]; then
+  awk -v a=$COMMENT_LINES \
+    'BEGIN {printf "%.1fk\n", a/1000}'
+fi
+
+# Show precentage of comments if --percentage is given.
+if [[ $* == *--percentage* ]]; then
+  awk -v a=$COMMENT_LINES -v b=$LINES_OF_CODE \
+    'BEGIN {printf "%.1f\n", 100*a/b}'
+fi

llmriddles/llms/__init__.py

@@ -0,0 +1,4 @@
+from .base import register_llm, get_llm_fn
+from .chatgpt import ask_chatgpt
+from .chatglm import ask_chatglm
+from .mistral import ask_mistral_7b_instruct

llmriddles/llms/base.py

@@ -0,0 +1,11 @@
+from typing import Callable, Dict
+
+_LLMS: Dict[str, Callable] = {}
+
+
+def register_llm(name: str, llm_ask_fn: Callable):
+    _LLMS[name] = llm_ask_fn
+
+
+def get_llm_fn(name: str) -> Callable:
+    return _LLMS[name]

llmriddles/llms/chatglm.py

@@ -0,0 +1,26 @@
+import zhipuai
+from .base import register_llm
+
+
+def ask_chatglm(message: str, api_key: str):
+    zhipuai.api_key = api_key
+
+    response = zhipuai.model_api.invoke(
+        model="chatglm_turbo",
+        prompt=[{
+            "role": "user",
+            "content": message
+        }],
+        top_p=0.7,
+        temperature=0.9,
+    )
+
+    response_msg = response['data']['choices'][0]['content']
+    # strip the front and end '"'
+    if len(response_msg) >= 2:
+        response_msg = response_msg[1:-1]
+
+    return response_msg
+
+
+register_llm('chatglm', ask_chatglm)

llmriddles/llms/chatgpt.py

@@ -0,0 +1,25 @@
+from functools import lru_cache
+
+from openai import OpenAI
+
+from .base import register_llm
+
+
+@lru_cache()
+def _get_openai_client(api_key):
+    return OpenAI(api_key=api_key)
+
+
+def ask_chatgpt(message: str, api_key: str):
+    client = _get_openai_client(api_key)
+
+    response = client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "user", "content": message}
+        ],
+    )
+    return response.choices[0].message.content.strip()
+
+
+register_llm('chatgpt', ask_chatgpt)

llmriddles/llms/llm_client.py

@@ -0,0 +1,54 @@
+import time
+import requests
+import logging
+import argparse
+
+
+class LLMFlaskClient:
+    def __init__(self, ip: str, port: int, max_retry: int = 3):
+        self.ip = ip
+        self.port = port
+
+        self.url_prefix_format = 'http://{}:{}/'
+        self.url = self.url_prefix_format.format(self.ip, self.port)
+        self.max_retry = max_retry
+
+        self.logger = logging.getLogger()
+        self.logger.addHandler(logging.StreamHandler())
+        self.logger.handlers[0].setFormatter(logging.Formatter("%(message)s"))
+
+    def _request(self, name: str, data: dict):
+        for _ in range(self.max_retry):
+            try:
+                self.logger.info(f'{name}\ndata: {data}')
+                response = requests.post(self.url + name, json=data).json()
+            except Exception as e:
+                self.logger.warning('error: ', repr(e))
+                time.sleep(1)
+                continue
+            if response['code'] == 0:
+                return response['output']
+            else:
+                raise Exception(response['error_msg'])
+        raise Exception("Web service failed. Please retry or contact with manager")
+
+    def run(self, message: str) -> str:
+        try:
+            return self._request('ask_llm_for_answer', {'user_text': message})
+        except Exception as e:
+            return f"Error: {repr(e)}"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--ip', required=True)
+    parser.add_argument('-p', '--port', required=True)
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+    if args.debug:
+        logging.getLogger().setLevel(logging.INFO)
+    else:
+        logging.getLogger().setLevel(logging.WARNING)
+
+    client = LLMFlaskClient(args.ip, args.port)
+    print(client.run('Please concatenate string "1+" and "1=3". Only give me the result without "".'))

llmriddles/llms/llm_server.py

@@ -0,0 +1,72 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from flask import Flask, request
+import argparse
+import logging
+
+
+class LLMInstance:
+
+    def __init__(self, model_path: str, device: str = "cuda"):
+
+        self.model = AutoModelForCausalLM.from_pretrained(model_path)
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path)
+        self.model.to(device)
+        self.device = device
+
+    def query(self, message):
+        try:
+            messages = [
+                {"role": "user", "content": message},
+            ]
+            encodeds = self.tokenizer.apply_chat_template(messages, return_tensors="pt")
+            model_inputs = encodeds.to(self.device)
+
+            generated_ids = self.model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+            decoded = self.tokenizer.batch_decode(generated_ids)
+
+            # output is the string decoded[0] after "[/INST]". There may exist "</s>", delete it.
+            output = decoded[0].split("[/INST]")[1].split("</s>")[0]
+            return {
+                'code': 0,
+                'ret': True,
+                'error_msg': None,
+                'output': output
+            }
+        except Exception as e:
+            return {
+                'code': 1,
+                'ret': False,
+                'error_msg': str(e),
+                'output': None
+            }
+
+
+def create_app(core):
+    app = Flask(__name__)
+
+    @app.route('/ask_llm_for_answer', methods=['POST'])
+    def ask_llm_for_answer():
+        user_text = request.json['user_text']
+        return core.query(user_text)
+
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-m', '--model_path', required=True, default='Mistral-7B-Instruct-v0.1', help='the model path of reward model')
+    parser.add_argument('--ip', default='0.0.0.0')
+    parser.add_argument('-p', '--port', default=8001)
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+    else:
+        logging.getLogger().setLevel(logging.INFO)
+    logging.getLogger().addHandler(logging.StreamHandler())
+    logging.getLogger().handlers[0].setFormatter(logging.Formatter("%(message)s"))
+
+    core = LLMInstance(args.model_path)
+    app = create_app(core)
+    app.run(host=args.ip, port=args.port)

llmriddles/llms/mistral.py

@@ -0,0 +1,22 @@
+from functools import lru_cache
+
+from .base import register_llm
+from .llm_client import LLMFlaskClient
+
+
+@lru_cache()
+def _get_mistral_7b_instruct_server(host: str, port: int):
+    from .llm_server import LLMInstance, create_app
+    core = LLMInstance('Mistral-7B-Instruct-v0.1')
+    app = create_app(core)
+    app.run(host=host, port=port)
+
+
+def ask_mistral_7b_instruct(message: str, **kwargs):
+    host, port = '0.0.0.0', 8001
+    _get_mistral_7b_instruct_server(host, port)
+    client = LLMFlaskClient(host, port)
+    return client.run(message).strip()
+
+
+register_llm('mistral-7b', ask_mistral_7b_instruct)

llmriddles/questions/__init__.py

@@ -0,0 +1,13 @@
+from .executor import QuestionExecutor
+from .level1 import __file__ as _level1_file_
+from .level2 import __file__ as _level2_file_
+from .level3 import __file__ as _level3_file_
+from .level4 import __file__ as _level4_file_
+from .level5 import __file__ as _level5_file_
+from .question import Question, register_question, list_ordered_questions
+
+_ = _level1_file_
+_ = _level2_file_
+_ = _level3_file_
+_ = _level4_file_
+_ = _level5_file_

llmriddles/questions/executor.py

@@ -0,0 +1,35 @@
+from typing import Tuple
+
+from .question import Question
+from ..llms import get_llm_fn
+
+
+class QuestionExecutor:
+    def __init__(self, question: Question, lang: str = 'cn', llm: str = 'chatgpt', llm_cfgs=None):
+        self.question = question
+        self.lang = lang
+        self.llm = llm
+        self.llm_cfgs = dict(llm_cfgs or {})
+
+    @property
+    def question_text(self):
+        return self.question.texts[self.lang]
+
+    @property
+    def question_name(self):
+        return self.question.names[self.lang]
+
+    def check(self, qs_text: str) -> Tuple[str, bool, str]:
+        answer_text = get_llm_fn(self.llm)(qs_text, **self.llm_cfgs)
+        correct, explanation = self.check_answer(qs_text, answer_text)
+        return answer_text, correct, explanation
+
+    def check_answer(self, user_text: str, answer_text: str) -> Tuple[bool, str]:
+        correct, explanation = self.question.checker(self.question_text, user_text, answer_text, self.lang)
+        if explanation is None:
+            if correct:
+                explanation = 'LLM的回答满足要求' if self.lang == 'cn' else 'Correct Answer From LLM'
+            else:
+                explanation = 'LLM的回答不满足要求' if self.lang == 'cn' else 'Wrong Answer From LLM'
+
+        return correct, explanation

llmriddles/questions/level1.py

@@ -0,0 +1,204 @@
+from .question import register_question
+
+
+def count_english_words(text: str):
+    return len(text.split(' '))
+
+
+def count_chinese_words(text: str):
+    return len(text)
+
+
+def check_if_chinese(text: str):
+    return all('\u4e00' <= char <= '\u9fff' for char in text)
+
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words) + len(english_words) + len(other_words)
+    else:
+        return len(chinese_words) + len(english_words)
+
+
+CN_TEXT_1 = """
+第一章第一题（初来乍到），请你构造一个问题使模型的回答是一字不差的“1+1=3”（不需要引号）。
+
+请在下面的输入框内填写你的构造并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 1, please construct a query for the model that will result in the exact answer "1+1=3" (no quotes needed).
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+    if answer_text == '1+1=3':
+        return True, None
+    else:
+        return False, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers=_checker_1,
+    name={'cn': '1-1 初来乍到', 'en': '1-1'},
+)
+
+CN_TEXT_2 = """
+第一章第二题（小试牛刀），请你输入三个字（及）以内的问题，使模型的回答在30个字以上。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_2 = """
+For the second question in chapter 1, please enter a question within three words so that the model’s answer is more than 30 words.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 3:
+        return False, "用户的问题长度应该在三个字及以内" if lang == 'cn' else 'Question should be within three words.'
+    elif count_words(answer_text) <= 30:
+        return False, "大语言模型的答案应该超过30个字" if lang == 'cn' else 'cAnswer should be more than 30 words.'
+    else:
+        return True, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_2,
+        'en': EN_TEXT_2,
+    },
+    checkers=_checker_2,
+    name={'cn': '1-2 小试牛刀', 'en': '1-2'},
+)
+
+CN_TEXT_3 = """
+第一章第三题（短说长话），请你输入一个字的问题，使模型的回答在100个字以上。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the third question in chapter 1, please enter a one-word question so that the model’s answer is more than 100 words.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 1:
+        return False, "用户的问题长度应该在一个字及以内" if lang == 'cn' else 'Question should be one word.'
+    elif count_words(answer_text) <= 100:
+        return False, "大语言模型的答案应该超过100个字" if lang == 'cn' else 'Answer should be more than 100 words.'
+    else:
+        return True, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers=_checker_3,
+    name={'cn': '1-3 短说长话', 'en': '1-3'}
+)
+
+CN_TEXT_4 = """
+第一章第四题（短说短话），请输入一个字的问题，使模型的回答字数小于20个字。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_4 = """
+For the fourth question in chapter 1, please enter a one-word question so that the model’s answer is less than 20 words.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 1:
+        return False, "用户的问题长度应该在一个字及以内" if lang == 'cn' else 'Question should be one word.'
+    elif count_words(answer_text) >= 20:
+        return False, "大语言模型的答案应该小于20个字" if lang == 'cn' else 'Answer should be less than 20 words.'
+    else:
+        return True, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_4,
+        'en': EN_TEXT_4,
+    },
+    checkers=_checker_4,
+    name={'cn': '1-4 短说短话', 'en': '1-4'},
+)
+
+# CN_TEXT_5 = """
+# 第一章第五题（回文不变），请输入一个本身不是回文串的问题，使无论正着问还是倒着问，模型的回答是一样的。
+
+# 请在下面的输入框内填写你的问题并点击按钮提交。
+# """
+
+# EN_TEXT_5 = """
+# For the fourth question in chapter 1, please enter a question that is not a palindrome string so that the model's answer is the same whether it is asked forward or backward.
+
+# Please enter your query below and click the submit button
+# """
+
+# def _checker_5(question_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+
+#     if count_words(question_text) > 0:
+#         return False, 'Question should be one word.'
+#     elif count_words(answer_text) >= 20:
+#         return False, 'Answer should be less than 20 words.'
+#     else:
+#         return True, None
+
+# register_question({
+#     'cn': CN_TEXT_5,
+#     'en': EN_TEXT_5,
+# }, _checker_5)

llmriddles/questions/level2.py

@@ -0,0 +1,329 @@
+import re
+from typing import Optional, Tuple
+
+import sympy
+
+from .question import register_question
+
+CN_TEXT_1 = """
+第二章第一题（质数长度），你需要提出一个字数是质数的问题，使回答的长度刚好是它的下一个质数。
+"""
+EN_TEXT_1 = """
+For the first question in chapter 2, You need to come up with a question that has a prime number of words, so the answer's length is exactly the next prime number.
+"""
+
+
+def _is_prime(v):
+    return sympy.isprime(v)
+
+
+def _next_prime(v):
+    while v:
+        v += 1
+        if _is_prime(v):
+            return v
+
+
+def _cn_checker_1(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    qs_length = len(user_text.strip())
+    if not _is_prime(qs_length):
+        return False, f'问题长度为{qs_length}，非质数'
+
+    answer_value = len(answer_text)
+    next_prime = _next_prime(qs_length)
+    if answer_value != next_prime:
+        return False, f'下一个质数为{next_prime}，但回答长度为{answer_value}'
+
+    return True, None
+
+
+def _en_words(text: str):
+    return len(re.findall(r'\w+', text))
+
+
+def _en_checker_1(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    qs_length = _en_words(user_text.strip())
+    if not _is_prime(qs_length):
+        return False, f'The question has a length of {qs_length}, which is not a prime number'
+
+    answer_value = _en_words(answer_text)
+    next_prime = _next_prime(qs_length)
+    if answer_value != next_prime:
+        return False, f'The next prime number is {next_prime}, but the answer\'s length is {answer_value}'
+
+    return True, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers={
+        'cn': _cn_checker_1,
+        'en': _en_checker_1,
+    },
+    name={'cn': '2-1 质数长度', 'en': '2-1'},
+    level=2
+)
+
+CN_TEXT_2 = """
+第二章第二题（越说越大），请输入一个大于一的正整数作为问题，使回答里包含至少比它大一千的数。
+"""
+EN_TEXT_2 = """
+For the second question in chapter 2, Please enter a positive integer greater than one as a question, so that the answer contains a number that is at least one thousand more than it.
+"""
+
+
+def _cn_checker_2(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'输入内容{user_text!r}，并非一个大于1的正整数'
+
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item >= value + 1000:
+            return True, f'检测到输出中数字{value_item}，满足要求'
+
+    return False, f'未在输出中检测到不少于{value + 1000}的数字'
+
+
+def _en_checker_2(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'You entered {user_text!r}, which is not a positive integer greater than 1'
+
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item >= value + 1000:
+            return True, f'Detected the number {value_item} in the output, which meets the requirement'
+
+    return False, f'Did not detect a number of at least {value + 1000} in the output'
+
+
+register_question(
+    {
+        'cn': CN_TEXT_2,
+        'en': EN_TEXT_2,
+    },
+    checkers={
+        'cn': _cn_checker_2,
+        'en': _en_checker_2,
+    },
+    name={'cn': '2-2 越说越大', 'en': '2-2'},
+    level=2
+)
+
+CN_TEXT_3 = """
+第二章第三题（越说越小），请输入一个大于一的正整数作为问题，使回答里包含至少10个不同的至少比它小一千的自然数。
+"""
+EN_TEXT_3 = """
+For the third question in chapter 2, Please enter a positive integer greater than one as a question, so that the answer contains at least 10 different natural numbers, each of them at least 1000 less than the question.
+"""
+
+
+def _cn_checker_3(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'输入内容{user_text!r}，并非一个大于1的正整数'
+
+    collected_values = []
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item <= value - 1000:
+            collected_values.append(value_item)
+
+    collected_values = sorted(set(collected_values))
+    if len(collected_values) >= 10:
+        return True, f'检测到{len(collected_values)}个不大于{value - 1000}的数字：{collected_values!r}'
+    else:
+        return False, f'检测到{len(collected_values)}个不大于{value - 1000}的数字,未达到10个：{collected_values!r}'
+
+
+def _en_checker_3(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'You entered {user_text!r}, which is not a positive integer greater than 1'
+
+    collected_values = []
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item <= value - 1000:
+            collected_values.append(value_item)
+
+    collected_values = sorted(set(collected_values))
+    if len(collected_values) >= 10:
+        return True, f'Detected {len(collected_values)} numbers not greater than {value - 1000}: {collected_values!r}'
+    else:
+        return False, f'Detected {len(collected_values)} numbers not greater than {value - 1000}, ' \
+                      f'but it doesn\'t reach 10: {collected_values!r}'
+
+
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers={
+        'cn': _cn_checker_3,
+        'en': _en_checker_3,
+    },
+    name={'cn': '2-3 越说越小', 'en': '2-3'},
+    level=2,
+)
+
+CN_TEXT_4 = """
+第二章第四题（警世名言），请在不提及1,4,5这三个字符的情况下让模型回答114514，不要有多余的字符。
+"""
+EN_TEXT_4 = """
+For the fourth question in chapter 2, Please make the model answer "114514" without mentioning the characters 1, 4, and 5, and without any extra characters.
+"""
+
+
+def _cn_checker_4(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    if "1" in user_text or "4" in user_text or "5" in user_text:
+        return False, '请不要在提问中出现1、4、5三个数字'
+
+    return answer_text.strip() == '114514', None
+
+
+def _en_checker_4(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    if "1" in user_text or "4" in user_text or "5" in user_text:
+        return False, 'Please don\'t include the numbers 1, 4, and 5 in the question'
+
+    return answer_text.strip() == '114514', None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_4,
+        'en': EN_TEXT_4,
+    },
+    checkers={
+        'cn': _cn_checker_4,
+        'en': _en_checker_4,
+    },
+    name={'cn': '2-4 警世名言', 'en': '2-4'},
+    level=2,
+)
+
+CN_TEXT_5 = """
+第二章第五题（开平方数），请输入一个大于一的平方数作为问题，使回答里包含它的平方根对应的数。
+"""
+EN_TEXT_5 = """
+For the fifth question in chapter 2, Please provide a square number greater than one as the question, and the answer should include the number corresponding to its square root.
+"""
+
+
+def _cn_checker_5(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+        if int(value ** 0.5) ** 2 != value:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'输入内容{user_text!r}，并非一个大于1的平方数'
+
+    sq = int(value ** 0.5)
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item == sq:
+            return True, f'检测到平方根数{sq}'
+    else:
+        return False, f'未检测到平方根数{sq}'
+
+
+def _en_checker_5(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    try:
+        value = int(user_text.strip())
+        if value <= 1:
+            raise ValueError
+        if int(value ** 0.5) ** 2 != value:
+            raise ValueError
+    except (TypeError, ValueError):
+        return False, f'The input, {user_text!r}, is not a square number greater than 1'
+
+    sq = int(value ** 0.5)
+    for value_item in re.findall('[-+]?\d+', answer_text):
+        value_item = int(value_item)
+        if value_item == sq:
+            return True, f'Detected square root: {sq}'
+    else:
+        return False, f'Square root {sq} not detected'
+
+
+register_question(
+    {
+        'cn': CN_TEXT_5,
+        'en': EN_TEXT_5,
+    },
+    checkers={
+        'cn': _cn_checker_5,
+        'en': _en_checker_5,
+    },
+    name={'cn': '2-5 开平方数', 'en': '2-5'},
+    level=2,
+)
+
+CN_TEXT_6 = """
+第二章第六题（得寸进狗），请提一个不包含“狗”这个字的10个字以内的问题，使回答中“狗”这个字出现的次数至少是问题字数的2倍。
+"""
+EN_TEXT_6 = """
+For the sixth question in chapter 2, Please ask a question in under 10 words without the word "dog" and ensure that the word "dog" appears at least twice in the answer for each word in the question.
+"""
+
+
+def _cn_checker_6(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    if len(user_text) > 10:
+        return False, '问题不得超过10个字'
+    if '狗' in user_text:
+        return False, '问题不得包含“狗”字'
+
+    dog_count = len(re.findall('狗', answer_text))
+    if dog_count >= len(user_text) * 2:
+        return True, f'“狗”字的出现次数为{dog_count}次'
+    else:
+        return False, f'“狗”字的出现次数为{dog_count}次，未达到{len(user_text) * 2}次'
+
+
+def _en_checker_6(question_text: str, user_text: str, answer_text: str) -> Tuple[bool, Optional[str]]:
+    q_words = re.findall(r'\w+', user_text.lower())
+    if len(q_words) > 10:
+        return False, 'The question must not exceed 10 words'
+    if any(word in {'dog', 'dogs'} for word in q_words):
+        return False, 'The question must not contain the word "dog" or "dogs"'
+
+    a_words = re.findall(r'\w+', answer_text.lower())
+    a_dog_count = sum(1 if word in {'dog', 'dogs'} else 0 for word in a_words)
+    if a_dog_count >= len(q_words) * 2:
+        return True, f'The word "dog" (or "dogs") appears {a_dog_count} times.'
+    else:
+        return False, f'The word "dog" (or "dogs") appears {a_dog_count} times, ' \
+                      f'which is less than {len(q_words) * 2} times.'
+
+
+register_question(
+    {
+        'cn': CN_TEXT_6,
+        'en': EN_TEXT_6,
+    },
+    checkers={
+        'cn': _cn_checker_6,
+        'en': _en_checker_6,
+    },
+    name={'cn': '2-6 得寸进狗', 'en': '2-6'},
+    level=2
+)

llmriddles/questions/level3.py

@@ -0,0 +1,165 @@
+from .question import register_question
+
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words) + len(english_words) + len(other_words)
+    else:
+        return len(chinese_words) + len(english_words)
+
+
+CN_TEXT_1 = """
+第三章第一题（自说自话），请你输入一个至少10个字的提问，使模型的回答和提问完全一样。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 3, please enter a question of at least ten words so that the model's answer is exatly the same with the question.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) < 10:
+        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
+
+    if answer_text == user_text:
+        return True, None
+    else:
+        return False, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers=_checker_1,
+    name={'cn': '3-1 自说自话', 'en': '3-1'},
+    level=3,
+)
+
+CN_TEXT_2 = """
+第三章第二题（话自说自），请你输入一个至少10个字的提问，使模型的回答是问题的反序。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_2 = """
+For the second question in chapter 3, please enter a question of at least ten words so that the model's answer is the reverse order of the question.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) < 10:
+        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
+
+    if lang == 'cn':
+        user_text = user_text[::-1]
+    else:
+        words = user_text.split()
+        words.reverse()
+        user_text = ' '.join(words)
+
+    if user_text == answer_text:
+        return True, None
+    else:
+        return False, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_2,
+        'en': EN_TEXT_2,
+    },
+    checkers=_checker_2,
+    name={'cn': '3-2 话自说自', 'en': '3-2'},
+    level=3,
+)
+
+CN_TEXT_3 = """
+第三章第三题（就是敢说），请你输入一个不包含“1”和“3”的提问，使模型的回答是一字不差的“1+1=3”（不需要引号）。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the third question in chapter 3, please enter a question not contain "1" or "3" that will result in the exact answer "1+1=3" (no quotes needed).
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if '1' in user_text or '3' in user_text:
+        return False, "用户的问题不能包含“1”或“3”" if lang == 'cn' else 'Question should not contain \"1\" or \"3\".'
+
+    if answer_text == '1+1=3':
+        return True, None
+    else:
+        return False, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers=_checker_3,
+    name={'cn': '3-3 就是敢说', 'en': '3-3'},
+    level=3,
+)
+
+# CN_TEXT_4 = """
+# 第三章第四题（回文协变），请你输入一个本身不是回文串的问题，使得正着问和倒着问时，模型的回答本身不是回文且也是逆序。
+
+# 请在下面的输入框内填写你的提问并点击按钮提交。
+# """
+
+# EN_TEXT_4 = """
+# For the fourth question in chapter 3, please enter a question that is not a palindrome string, so that the model's answer is also not a palindrome and is in reverse order when asked forward or backward.
+
+# Please enter your query below and click the submit button
+# """
+
+# def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
+#     pass
+
+# register_question({
+#     'cn': CN_TEXT_4,
+#     'en': EN_TEXT_4,
+# }, _checker_4, level=3)

llmriddles/questions/level4.py

@@ -0,0 +1,104 @@
+import re
+
+from .question import register_question
+
+
+def check_if_is_number(text: str):
+    try:
+        int(text)
+        return True
+    except ValueError:
+        return False
+
+
+def get_all_numbers_in_a_sentence(text: str):
+    return [int(i) for i in re.findall(r'\d+', text)]
+
+
+# CN_TEXT_1 = """
+# 第四章第一题，请构造一个问题A，它的回复是不同于A的B，然后以B提问能再次得到A。
+
+# 请在下面的输入框内填写你构造并点击按钮提交。
+# """
+
+# EN_TEXT_1 = """
+# For the first question in chapter 4, please make a question A whose answer is B that is different from A, and then ask B to get A again.
+
+# Please enter your query below and click the submit button
+# """
+
+
+# def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+#     user_text = user_text.strip()
+#     pass
+
+# register_question({
+#     'cn': CN_TEXT_1,
+#     'en': EN_TEXT_1,
+# }, _checker_1, level=4)
+
+
+# CN_TEXT_2 = """
+# 第四章第二题，
+
+# 请在下面的输入框内填写你构造并点击按钮提交。
+# """
+
+# EN_TEXT_2 = """
+# For the second question in chapter 4,
+
+# Please enter your query below and click the submit button
+# """
+
+
+# def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+#     user_text = user_text.strip()
+#     pass
+
+# register_question({
+#     'cn': CN_TEXT_2,
+#     'en': EN_TEXT_2,
+# }, _checker_2, level=4)
+
+
+CN_TEXT_3 = """
+第四章第一题（自然之密），请输入一个大于一的正整数作为问题，使回答里包含和它刚好相差1的数。
+
+请在下面的输入框内填写你构造并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the first question in chapter 4, please enter a positive integer greater than one as the question so that the answer contains a number that is exactly 1 different from it.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+    if not check_if_is_number(user_text):
+        return False, "问题应该是一个正整数" if lang == 'cn' else 'Question should be a positive integer.'
+    elif int(user_text) == 1:
+        return False, "问题应该是一个大于1的正整数" if lang == 'cn' else 'Question should be a positive integer greater than 1.'
+    elif int(user_text) - 1 not in get_all_numbers_in_a_sentence(answer_text) or int(
+            user_text) + 1 not in get_all_numbers_in_a_sentence(answer_text):
+        return False, "回答中应该包含一个与问题相差1的数字" if lang == 'cn' else 'Answer should contain a number that is exactly 1 different from the question.'
+    else:
+        return True, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers=_checker_3,
+    name={'cn': '4-3 自然之密', 'en': '4-3'},
+    level=4,
+)

llmriddles/questions/level5.py

@@ -0,0 +1,64 @@
+from .question import register_question
+
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words) + len(english_words) + len(other_words)
+    else:
+        return len(chinese_words) + len(english_words)
+
+
+CN_TEXT_1 = """
+第五章第一题（口是心非），请构造一个不少于十个字的问题，使得回答中不包含问题中的任意字符。
+
+请在下面的输入框内填写你的提问并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 5, Please construct a question of no less than ten words so that the answer does not contain any of the characters in the question.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) < 10:
+        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
+
+    if answer_text == user_text:
+        return True, None
+    else:
+        return False, None
+
+
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers=_checker_1,
+    name={'cn': '5-1 口是心非', 'en': '5-1'},
+    level=5,
+)

llmriddles/questions/question.py

@@ -0,0 +1,52 @@
+import collections.abc
+from dataclasses import dataclass
+from typing import Union, Mapping, Literal, Callable, Tuple, List, Optional
+
+LangTyping = Literal['en', 'cn']
+MultiLangCheckerTyping = Callable[[str, str, str, str], Tuple[bool, Optional[str]]]
+SingleLangCheckerTyping = Callable[[str, str, str], Tuple[bool, Optional[str]]]
+
+
+@dataclass
+class Question:
+    texts: Mapping[str, str]
+    checker: MultiLangCheckerTyping
+    names: Mapping[str, str]
+    level: int
+
+
+_KNOWN_PROBLEMS = []
+
+
+def register_question(text: Union[Mapping[str, str], str],
+                      checkers: Union[Mapping[str, SingleLangCheckerTyping], MultiLangCheckerTyping],
+                      name=Union[Mapping[str, str], str],
+                      level: int = 1, default_lang='cn'):
+    if isinstance(checkers, collections.abc.Mapping):
+        _origin_checkers = checkers
+
+        def _integrated_checker(question_text: str, user_text: str, answer_text: str, lang: str):
+            return _origin_checkers[lang](question_text, user_text, answer_text)
+
+        checker: MultiLangCheckerTyping = _integrated_checker
+    else:
+        checker: MultiLangCheckerTyping = checkers
+
+    if isinstance(text, str):
+        texts = {default_lang: text}
+    else:
+        texts = text
+
+    if isinstance(name, str):
+        names = {default_lang: name}
+    else:
+        names = name
+
+    _KNOWN_PROBLEMS.append(Question(texts, checker, names, level))
+
+
+def list_ordered_questions() -> List[Question]:
+    return [
+        problem for _, problem in
+        sorted(enumerate(_KNOWN_PROBLEMS), key=lambda x: (x[1].level, x[0]))
+    ]

requirements-dev.txt

@@ -0,0 +1,11 @@
+hbutils>=0.9.1
+tqdm
+requests>=2.20
+gradio==4.1.1
+openai>=1
+zhipuai
+sympy
+flask
+transformers
+torch
+huggingface-hub

requirements.txt

@@ -0,0 +1,7 @@
+hbutils>=0.9.1
+tqdm
+requests>=2.20
+gradio==4.1.1
+openai>=1
+zhipuai
+sympy