Results - Document Visual Question Answering

method: Human Performance2022-03-02

Authors: DocVQA Organizers

Affiliation: CVC-UAB, IIIT Hyderabad

Description: Human performance as reported in InfographicVQA paper

@InProceedings{Mathew_2022_WACV, author = {Mathew, Minesh and Bagal, Viraj and Tito, Rub\`en and Karatzas, Dimosthenis and Valveny, Ernest and Jawahar, C.V.}, title = {InfographicVQA}, booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision (WACV)}, month = {January}, year = {2022}, pages = {1697-1706} }

method: InternVL2.5-78B-MPO (generalist)2024-12-24

Authors: InternVL team

Affiliation: Shanghai AI Laboratory & Tsinghua University

Email: wangweiyun@pjlab.org.cn

Description: InternVL2.5-MPO: Enhancing the Reasoning Ability of Multimodal Large Language Models via Mixed Preference Optimization

@article{wang2024mpo, title={Enhancing the Reasoning Ability of Multimodal Large Language Models via Mixed Preference Optimization}, author={Wang, Weiyun and Chen, Zhe and Wang, Wenhai and Cao, Yue and Liu, Yangzhou and Gao, Zhangwei and Zhu, Jinguo and Zhu, Xizhou and Lu, Lewei and Qiao, Yu and Dai, Jifeng}, journal={arXiv preprint arXiv:2411.10442}, year={2024} }

@article{chen2024expanding, title={Expanding Performance Boundaries of Open-Source Multimodal Models with Model, Data, and Test-Time Scaling}, author={Chen, Zhe and Wang, Weiyun and Cao, Yue and Liu, Yangzhou and Gao, Zhangwei and Cui, Erfei and Zhu, Jinguo and Ye, Shenglong and Tian, Hao and Liu, Zhaoyang and others}, journal={arXiv preprint arXiv:2412.05271}, year={2024} }

Source code

method: InternVL2-Pro (generalist)2024-06-30

Authors: InternVL team

Affiliation: Shanghai AI Laboratory & Sensetime & Tsinghua University

Email: czcz94cz@gmail.com

Description: InternVL Family: Closing the Gap to Commercial Multimodal Models with Open-Source Suites —— A Pioneering Open-Source Alternative to GPT-4V

Demo: https://internvl.opengvlab.com/
Code: https://github.com/OpenGVLab/InternVL
Model: https://huggingface.co/OpenGVLab/InternVL-Chat-V1-5

@article{chen2023internvl, title={Internvl: Scaling up vision foundation models and aligning for generic visual-linguistic tasks}, author={Chen, Zhe and Wu, Jiannan and Wang, Wenhai and Su, Weijie and Chen, Guo and Xing, Sen and Muyan, Zhong and Zhang, Qinglong and Zhu, Xizhou and Lu, Lewei and others}, journal={arXiv preprint arXiv:2312.14238}, year={2023} }

@article{chen2024far, title={How Far Are We to GPT-4V? Closing the Gap to Commercial Multimodal Models with Open-Source Suites}, author={Chen, Zhe and Wang, Weiyun and Tian, Hao and Ye, Shenglong and Gao, Zhangwei and Cui, Erfei and Tong, Wenwen and Hu, Kongzhi and Luo, Jiapeng and Ma, Zheng and others}, journal={arXiv preprint arXiv:2404.16821}, year={2024} }

Source code

Ranking Table

Description Paper Source Code

			Answer type				Evidence					Operation
Date	Method	Score	Image span	Question span	Multiple spans	Non span	Table/List	Textual	Visual object	Figure	Map	Comparison	Arithmetic	Counting
2022-03-02	Human Performance	0.9718	0.9745	0.9777	0.9335	0.9716	0.9780	0.9789	0.9770	0.9699	0.9433	0.9712	0.9837	0.9544
2024-12-24	InternVL2.5-78B-MPO (generalist)	0.8428	0.8765	0.8753	0.6977	0.7357	0.8313	0.9247	0.8398	0.8229	0.7338	0.7280	0.8812	0.5865
2024-06-30	InternVL2-Pro (generalist)	0.8334	0.8681	0.8929	0.7350	0.6969	0.8335	0.9260	0.7757	0.8093	0.7186	0.7301	0.8584	0.5368
2024-09-25	Molmo-72B	0.8186	0.8513	0.8827	0.6821	0.7041	0.8184	0.9136	0.8062	0.7945	0.6960	0.7054	0.8188	0.5930
2024-12-13	DeepSeek-VL2	0.7814	0.8189	0.8010	0.6989	0.6363	0.7935	0.9041	0.7371	0.7434	0.6327	0.6206	0.7282	0.5326
2024-04-27	InternVL-1.5-Plus (generalist)	0.7574	0.7989	0.8124	0.6425	0.5987	0.7544	0.8733	0.7306	0.7234	0.6216	0.6065	0.7386	0.4623
2024-05-31	GPT-4 Vision Turbo + Amazon Textract OCR	0.7191	0.7575	0.7795	0.6591	0.5553	0.7183	0.8201	0.6696	0.6904	0.6926	0.5815	0.6759	0.4281
2024-11-01	MLCD-Embodied-7B: Multi-label Cluster Discrimination for Visual Representation Learning	0.6998	0.7330	0.7930	0.5955	0.5564	0.6951	0.8271	0.6654	0.6614	0.5495	0.5523	0.6350	0.4905
2023-11-15	SMoLA-PaLI-X Specialist Model	0.6621	0.7166	0.7252	0.5838	0.4292	0.6448	0.8261	0.6714	0.6110	0.5065	0.5238	0.5054	0.3506
2024-02-10	ScreenAI 5B	0.6590	0.7162	0.7247	0.5734	0.4140	0.6525	0.8315	0.5968	0.6020	0.4467	0.4815	0.5303	0.3000
2023-12-07	SMoLA-PaLI-X Generalist Model	0.6556	0.7107	0.7228	0.5642	0.4197	0.6200	0.8237	0.6710	0.6095	0.5246	0.5159	0.4988	0.3372
2021-04-11	Applica.ai TILT	0.6120	0.6765	0.6419	0.4391	0.3832	0.5917	0.7916	0.4545	0.5654	0.4480	0.4801	0.4958	0.2652
2024-07-22	Snowflake Arctic-TILT 0.8B	0.5695	0.6274	0.6074	0.4123	0.3653	0.5478	0.7530	0.4204	0.5109	0.4410	0.4350	0.5042	0.2238
2023-08-20	PaLI-X (Google Research, Single Generative Model)	0.5477	0.5940	0.6950	0.4122	0.3534	0.5145	0.6891	0.6373	0.5040	0.4013	0.4290	0.4053	0.3091
2022-03-03	InfographicVQA paper model	0.2720	0.3278	0.2386	0.0450	0.1371	0.2400	0.3626	0.1705	0.2551	0.2205	0.1836	0.1559	0.1140

Inactive evaluations

method: Human Performance2022-03-02

method: InternVL2.5-78B-MPO (generalist)2024-12-24

method: InternVL2-Pro (generalist)2024-06-30

Ranking Table

Ranking Graphic