searchindex.js

Search.setIndex({docnames:["architecture/alpa_compiler_walk_through","architecture/intra_op_solver","architecture/overview","architecture/parallelism-view-and-rationale","benchmark/benchmark","developer/developer_guide","gallery/tutorials/README","index","install","publications/publications","tutorials/alpa_on_slurm","tutorials/alpa_vs_pmap","tutorials/faq","tutorials/icml_big_model_tutorial","tutorials/index","tutorials/opt_serving","tutorials/perf_tuning_guide","tutorials/pipeshard_parallelism","tutorials/quickstart","tutorials/sg_execution_times"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["architecture/alpa_compiler_walk_through.rst","architecture/intra_op_solver.rst","architecture/overview.rst","architecture/parallelism-view-and-rationale.rst","benchmark/benchmark.rst","developer/developer_guide.rst","gallery/tutorials/README.rst","index.rst","install.rst","publications/publications.rst","tutorials/alpa_on_slurm.rst","tutorials/alpa_vs_pmap.rst","tutorials/faq.rst","tutorials/icml_big_model_tutorial.rst","tutorials/index.rst","tutorials/opt_serving.rst","tutorials/perf_tuning_guide.rst","tutorials/pipeshard_parallelism.rst","tutorials/quickstart.rst","tutorials/sg_execution_times.rst"],objects:{},objnames:{},objtypes:{},terms:{"0":[5,8,10,11,12,15,17,18,19],"00":[10,19],"000":[17,19],"000gb":[],"001":[11,17],"00105005":[],"00115792":[],"00148469":[],"00149446":[],"00153695":[],"00156432":[],"00159595":[],"00162239":[],"00178525":[],"00179476":[],"0019203":[],"0019856":[],"00198701":[],"001gb":[],"002":17,"00202494":[],"00203413":[],"00232795":[],"00238567":[],"00239089":[],"00245696":[],"00256801":[],"00274867":[],"00275159":[],"00278576":[],"00279755":[],"00297202":[],"002gb":[],"003":17,"00306678":[],"00313377":[],"003gb":[],"004":17,"00438674":[],"00445016":[],"00477897":[],"00477968":[],"004gb":[],"005":[],"00510118":[],"005gb":[],"006":17,"006gb":[],"007":17,"007gb":[],"008":17,"01":[],"012":17,"012gb":[],"02":[],"03":[17,19],"031":17,"032":17,"035":17,"036":17,"04":[],"05":[8,19],"050":17,"052":17,"058":17,"06":[],"060":17,"063":17,"063gb":[],"064":17,"065":17,"066":17,"067":17,"068":17,"069":17,"07":[],"073":18,"078gb":[],"08":[],"09":[],"094gb":[],"096":17,"097gb":[],"098":17,"098gb":[],"099gb":[],"0x7f759f346b50":[],"0xd":8,"0xf":8,"1":[1,2,4,10,15,17,18],"10":[10,18],"1000":15,"1014":[],"102":17,"1024":[15,18],"105":[],"109gb":[],"11":[8,10,15],"112":17,"114":17,"116":17,"12":8,"122":[],"1223":[],"125":17,"125gb":[],"125m":15,"126":17,"127":17,"129":17,"13":17,"130":17,"132":17,"133":17,"1333":[],"134":17,"1342":17,"14":5,"148":[],"15":[],"150":18,"1500":15,"1525":17,"156gb":[],"159":17,"16":[0,10,15,17],"1656":[],"1668":[],"16b":7,"16gb":10,"16xlarg":[2,15],"17":8,"172":17,"174":17,"175b":7,"176b":7,"179":[],"181de4f5577a72c9b30525ed3da09e5b2138cc2c":1,"188":17,"189":17,"190":17,"191":17,"192gb":[],"1939":[],"193gb":[],"194gb":[],"195":17,"1e":[17,18],"1e3":18,"1gb":10,"2":[0,1,2,5,10,15,17,18],"20":[8,17],"202":17,"2022":[8,9,13],"2023":[9,17],"2032":[],"2048":[17,18],"21":8,"211":[],"2134":[],"22":[2,7,8,18],"221":17,"223":17,"224":[],"226":17,"2289":[],"23":15,"230":17,"234":[],"24":[],"242":[],"25":17,"250":17,"250gb":[],"251":17,"251gb":[],"252":17,"252gb":[],"254":17,"2540":[],"256":15,"257":17,"26":[],"264":17,"266":17,"27":[17,18],"280":[],"29":17,"294":[],"2b":15,"2d":0,"2f":18,"3":[2,8,10,15,17,18],"30":[],"302":[],"30b":15,"31":17,"314":17,"316":17,"32":[5,17],"33":18,"3363":[],"34":[],"346":17,"348":17,"35":8,"350gb":15,"36":17,"37":[17,18],"377":17,"379":17,"38":18,"383gb":[],"384gb":[],"3873":[],"4":[0,2,8,15,17,18],"40":[],"4096":[],"43":[],"43493":[],"439":17,"441":17,"444":17,"45":[],"451":17,"452":8,"459":17,"46":[],"461":8,"47":[],"496":8,"4x8":2,"5":[8,10,17,18],"50":18,"500gb":[],"501gb":[],"502gb":[],"512":15,"52":18,"53":[],"55":18,"56":17,"560m":15,"57":[8,18],"5e":[17,18],"612":[],"621":[],"6379":17,"664":17,"664gb":[],"66b":15,"67":[],"6789":10,"688":17,"689":17,"69":[],"691":17,"694":17,"7":[8,10],"700":17,"700gb":15,"71":17,"710782":8,"712":[],"721260d122f096040762b2d226b37e8ab23f74b8":1,"73":[],"745":[17,19],"75":17,"76":18,"763gb":[],"76976db8412b60d322c680a5822116ba6f2f762a":8,"78":[17,18],"79":17,"7b":15,"8":[2,4,8,10,15,17,18],"8001":15,"806":[],"80gb":15,"8265":17,"8266":[],"83":8,"85":18,"858":[],"87":[],"9":[8,10],"90":[],"919":[],"925":[],"944":[],"992":15,"break":1,"byte":15,"case":[0,2,10,15,16,18],"catch":8,"class":[0,2,17,18],"default":[8,10],"do":[0,2,5,8,10,15,16,18],"export":10,"final":2,"function":[0,16,17,18],"import":[0,8,15],"int":[0,17,18],"long":8,"new":[0,2,5,8,15],"public":5,"return":[0,10,17,18],"static":2,"switch":8,"throw":15,"true":15,"try":[2,8,10,15,16],"while":[0,8,10,18],A:[0,2,10,11,15,17],As:[8,10,15,18],At:[2,10],By:[10,18],For:[0,2,8,10,12,15,17],IFS:10,If:[5,8,10,11,12,15,16,17],In:[0,2,5,10,11,15,16,17,18],It:[0,2,8],No:8,Not:15,On:[8,9,10,11],The:[0,1,2,4,5,8,10,11,15,16,17,18],Then:[0,8,10,15,17],There:[0,15],These:[2,8,10,16],To:[0,8,10,16,17,18],With:[2,11,15,17],__array__:[],__call__:[0,17,18],_asarrai:[],_get_buffers_with_local_id:[],_get_full_batch_apply_grad:0,_multiarray_umath:8,_privat:[],_valu:[],a100:15,abl:15,abort:8,about:[2,8,15],abov:[0,2,15,17,18],acceler:[0,15,18],access:[8,10,15,16],accord:[0,18],accordingli:16,accumul:[0,16,17],accumulate_grad:0,accumulated_grad:0,achiev:[11,18],acitv:[],activ:[10,15,18],actor:[2,8],actual_st:18,ad:[0,10],adam:[16,17,18],add:[0,10],add_bos_token:15,addit:[0,5,8,15,17],addition:[0,2,5],addr:10,address:[8,10,15,17],adf:[],adjac:2,adjust:8,advanc:[11,12,13,16,17,18],advantag:15,affect:16,after:[0,1,8,10,18],again:[8,17],against:8,ai:2,aim:2,algorithm:[0,2,16,17],all:[0,1,2,8,10,14,15,17,18],all_gath:18,alloc:16,allow:10,along:2,alpa:[1,2,4,9,13,16,17,19],alpa_cost:18,alpa_debug_print_as_strategi:[1,16],alpa_environ:10,alpa_execut:18,alpa_multinode_test:10,alpa_train_step:18,alpa_uninode_test:10,alpa_vs_pmap:[11,19],alpaserv:9,alreadi:[8,10,15],also:[0,2,8,10,11,15,16,17,18],altern:17,although:18,alwai:0,among:11,an:[0,2,4,8,10,11,15,16],analys:[16,18],analysi:[0,1],analyz:[2,11,18],ani:[2,8,18],annot:0,anoth:[2,8,10,15],api:[0,8,18],app:15,appear:10,appli:[2,10,12,15,17,18],applic:8,apply_fn:[0,17,18],apply_grad:0,apply_gradi:[0,17,18],approach:2,appropri:15,approxim:15,ar:[0,1,2,8,10,11,12,13,15,16,17,18],arbitrari:0,architectur:[15,16],arg:[],argument:[0,8,10,15,16,18],around:15,arrai:18,art:18,articl:11,as_config:[],as_instanceof_caus:[],asarrai:[],asdf:[],ask:[7,10],assert_allclos:[17,18],assign:[0,2,10,11,18],assum:10,asum:10,atol:[17,18],attach:11,auto:[1,16,17],auto_pipeline_actual_st:17,auto_pipeline_train_step:17,auto_shard:1,auto_sharding_dot_handl:1,auto_sharding_util:1,autograd:0,autolayeropt:[16,17],autom:[9,11,17],automat:[0,10,11,15,16,18],autoshard:0,autosharding_option_dict:17,autotoken:15,avail:[8,10,13,15,17],available_memori:17,avoid:16,aw:[2,4,15],ax:18,axi:2,axis_nam:18,b:[17,18],back:[0,8,18],backend:15,background:[10,13],backward:[0,8,17],bandwidth:2,base:[0,1,2,15,17],bash:10,basic:[1,16,17],batch:[0,1,15,16,17,18],batch_decod:15,batch_siz:[17,18],bazel:8,bazel_opt:8,becaus:[0,5,8,15,16,17,18],been:8,befor:[0,1,2,8,10,12,15],begin:18,behav:0,behavior:16,behind:9,being:[17,18],below:[0,1,2,4,5,8,10,11,15],benchmark:[5,18],benchmark_func:18,besid:11,best:[11,16,18],better:[11,17,18],between:[0,2,7,8,14,17,19],big:[7,15,16],bigscienc:15,billion:[4,11],bin:10,binari:2,bio:8,blob:[1,16],block:[10,16],blog:2,bloom:7,bodi:[17,18],boost:15,both:[0,7,11,14,18,19],boundari:[0,17],bring:2,browser:15,buf:[],buffer:[],build:[5,8,15],build_jaxlib:[5,8],built:18,c:[1,5,8,17],cach:[8,15],calcul:0,call:[0,1,2,17,18],can:[0,1,2,4,5,8,10,11,12,15,16,17,18],cannot:[8,11,15],capit:15,captur:0,categori:2,caus:8,caveat:12,cc:[1,8],cd865615b9b518bc507fbdc71dc44c7cc76618ac:1,cd:[8,15],chang:[10,11],check:[1,5,15],checkout:8,checkpoint:[0,15],checkpoint_last:15,chen:9,chex:8,choic:16,choos:8,choosen:1,chunk:[10,15],citi:15,ckpt:15,clean:8,click:[11,17,18],client:[8,15],client_mode_hook:[],clone:[8,15],cluster:[0,4,8,10,12,15,16,18],cluster_layers_and_slice_mesh:0,code:[0,2,7,8,11,14,15,17,18],codegen:7,collabor:9,collect:[10,15],com:[1,8,13,15,16],combin:[10,15,16,17,18],come:5,command:[5,8,10,16,17],commandlin:10,commit:[0,5],commod:15,common:[8,10,11,18],commun:[2,9,18],compact:[0,17,18],compar:[0,18],comparison:11,compat:[8,10,15],compil:[1,5,7,8,15,17,18],compile_pipeshard_execut:0,compile_pipeshard_executable_intern:0,complet:0,complic:[1,16],compon:1,componen:2,comput:[0,11,16,17,18],compute_cost:17,compute_grad:0,compute_grad_input:0,compute_grad_to_accumulate_grad:0,concat:0,concaten:16,concept:[13,17],conda:10,config:17,configur:[12,16,17],connect:[8,10,15],consist:[2,17],consolid:15,constant:15,constraint:15,construcion:17,construct:[16,17],contain:[1,2,8,10,17],contrari:11,contrast:18,contributor:5,control:[8,15,16,17],convert:[10,18],copi:17,core:[2,8],correct:[0,17,18],correctli:[8,10,16],correspond:[0,2,8,15,17],cost:[1,17,18],could:8,cover:[10,13],cpu:[8,10,16,17],creat:[17,18],create_donation_map:0,cross:2,ctrl:10,cu113:15,cuda111:8,cuda114:[],cuda11x:8,cuda:[10,15],cuda_vers:8,cudnn805:8,cudnn:[8,10],cudnn_vers:8,cupi:8,cupy_backend:8,cupyx:[],current:16,custom:[0,2,8],d987466b444ac0dbb9dd8c841bb0d960aed94fdd:[],d:[8,10],danyang:9,dashboard:17,data:[0,2,16,17],data_parallel_execut:18,dataparallel:16,dataset:17,deactiv:10,debug:[0,16],decor:[0,18],deep:[9,18],deepcopi:17,deeper:2,def:[0,17,18],defin:[0,10,15,17,18],definit:0,demo:13,dens:[0,17,18],dense_0:18,depend:[2,8,15,16,18],deploi:10,deriv:0,describ:[2,9],descript:2,design:[7,8,13,15],despit:2,detail:[0,8,11,15,17],detect:16,dev:8,dev_instal:8,develop:[8,9],devic:[0,11,16],device_indic:[],device_mesh:[],device_put_repl:18,device_put_shard:18,devicearrai:18,devices_per_nod:17,diagram:2,did:[15,16],differ:[0,2,7,10,14,15,16,18,19],dim:[1,17,18],dimension:17,direct:8,directli:[0,10],directori:10,disabl:8,disable_tqdm_glob:17,discuss:8,disk:15,disribut:17,dist:8,distinguish:[0,2],distribut:[0,2,7,9,11,14,15,18,19],distributedarrai:16,distributedli:18,dive:2,dma:8,do_sampl:15,doc:5,docstr:12,document:[0,2,5,8],doe:[16,18],doesn:0,don:10,donat:[0,18],donate_argnum:18,done:[0,10],doubl:8,downgrad:8,download:[8,11,14,15,17,18],dram:8,drive:2,driver:[8,15,16,17],dropvar:0,dtype:16,due:[8,18],dump:[8,16],dump_debug_info:16,dure:2,dynam:2,e:[0,8,9,10,15,17],each:[0,1,2,8,10,15,17],easier:16,echo:10,edg:1,ef:[],effect:8,effici:[4,15,17,18],either:[8,15],element:16,elementwis:1,els:[10,18],embed:0,enabl:[8,16,17],enable_cuda:8,encod:15,encoder_chunk_s:15,end:[0,11,15],enough:[8,18],ensur:17,entail:2,entir:15,entranc:1,environ:[1,8,12,16],equal:17,equat:0,equival:10,eric:9,error:[5,16],especi:16,etc:[2,15],eventu:0,everi:5,exactli:[0,2,10],exampl:[0,2,5,8,10,11,12,14,15,16,17,18],exce:10,except:15,execut:[0,2,10,15,16,17,19],exist:[2,8,17],expect:[10,16],expected_st:[17,18],experiment:[5,8],expertis:18,explain:2,explan:4,explicitli:8,express:2,extern:8,extra:[8,15],extract:2,f:[8,18],facebook:15,fact:2,fail:[8,15,16],fall:2,fals:15,familiar:[12,15],fanci:15,faq:7,fastapi:15,faster:18,featur:[0,5,11,17,18],fetch:16,few:[2,8,16],fi:10,field:10,figur:[0,2,4,15],file:[1,8,10,15,19],fill:[15,17],find:[8,10,15,18],finish:10,first:[0,2,8,10,15,17,18],fit:[15,17],fix:[0,8,16],flatten:0,flax:[0,17,18],flexflow:2,flexibl:[15,17],flow:8,focus:17,folder:15,follow:[0,2,5,8,10,15,17],follw:[],forc:17,force_batch_dim_to_mesh_dim:17,fork:[5,7],form:[0,2,11,15,17],forward:[0,8,17],forward_stage_layer_id:17,found:[0,4,8,10],four:[2,10],framework:[17,18],free:15,freed:18,frequent:[7,16],from:[0,1,2,9,10,15,16,17,18],from_pretrain:15,frontend:15,full:[0,2,11,17,18],fullfil:2,fulli:[15,18],func:16,functool:18,functorch:8,further:2,g:[0,10,15],galleri:[11,14,17,18],gb:[15,17,18],gcc:8,gener:[0,2,11,14,16,17,18],generate_sharded_xla_computations_argu:0,generated_str:15,get:[10,17],get_buff:[],get_execut:18,get_last_dp_result:16,get_model:15,get_remote_buff:[],get_total_allocation_s:[16,18],git:[5,8,15],github:[1,8,15,16],give:10,given:[15,17,18],global:[1,16,18],go:[10,11,15],gonzalez:9,good:15,googl:[2,5,13],got:[10,15],gpt:15,gpu:[2,10,15,16,17,18],gpus_per_nod:10,grab:10,grad:[0,16,17,18],grad_l1_x:0,grad_l2_x:0,grad_x:0,gradient:[0,16,17,18],gradual:16,graph:[0,11,16,17,18],gre:10,greatli:15,ground:[17,18],group:[2,12],gspmd:[0,2],gt:10,guarante:2,guid:[8,10,15],guidanc:16,ha:[0,8,17,18],hao:9,hardwar:15,hash:5,have:[2,8,10,15,16],head:[8,10,12,15],head_nod:10,head_node_ip:10,help:[1,8,13],henc:[2,10,15,17],here:[0,8,10,11,15,17,18],hexu:9,hidden_dim:[0,17,18],hierarchi:2,high:[2,18],higher:8,hlo:[0,1,2,16,17],hlomodul:0,ho:9,home:[],host:[5,15],hostnam:10,hour:15,hous:15,how:[0,2,5,10,15,17],howev:[5,8,11,15,16,18],html:8,http:[1,8,13,15,16,17],huang:9,huggingfac:15,hyperparamet:17,i:[0,10,17,18],icml:7,idea:[0,9],ident:0,illustr:[2,10],ilp:1,immedi:8,implement:[2,18],importerror:8,improv:2,includ:[0,2,8,9,10,16,17,18],incompat:8,increas:16,index:[8,15,16],indic:[0,8],individu:[0,2],inf:[],infer:[0,15],infiniband:15,info:17,inform:8,ingest:15,init:[5,17,18],initi:18,input:[0,2,8,18],input_id:15,insert:0,insid:10,instal:[5,7,15],install_librari:[],install_require_list:8,instanc:[10,15],instead:[0,10],institut:9,instruct:[0,1,2,4,8,10,15],inter:[0,9,17],interfac:[15,16],interfer:8,intermedi:[0,2,16,18],intern:18,interpret:10,intra:[0,7,9,16,17],introduc:[2,8],invar:0,invar_s:17,involv:2,io:8,iommu:8,ion:9,ip:[10,15],ip_head:10,ipv4:10,ipv6:10,ipynb:[11,17,18],issu:[0,8,10,16],iter:10,its:[0,2,15],j:17,jax:[0,7,14,15,16,17,19],jaxlib:[5,18],jaxpr:[0,2,17],jin:9,jinja2:15,jit:18,jit_train_step:18,jnp:[0,17,18],job:8,job_numb:10,join:15,joseph:9,jump:10,jupyt:[11,14,17,18],just:[10,16],k1:[17,18],k2:[17,18],k:[1,17],kei:16,kernel:18,kernel_np:18,keyerror:[],kind:0,knois:[17,18],know:[5,8,18],knowledg:16,known:8,ksampl:[17,18],kwarg:[],l1016:1,l172:1,l1:0,l2124:1,l2259:1,l2:0,l327:1,l408:1,l588:1,l616:[],l857:1,l967:1,languag:[0,15],larg:[7,10,11,15,16,18],larger:[16,18],largest:15,last:0,latenc:[2,17],later:[8,10,15,18],latest:[0,5,8,15],launch:[8,16],launch_model_work:15,launch_websit:15,lax:18,layer:[16,17],layer_input:0,layer_level_jaxpr_transform:0,layer_num:[16,17],layer_opt:[0,16,17],layer_output:0,lead:16,learn:[2,9,18],learning_r:[17,18],least:15,len:18,length:15,less:[11,18],let:[0,17],level:[2,18],li:9,lianmin:9,lib:8,librari:[8,15],like:[0,2,5,8,10,12,15,16],limit:10,line:[10,11],linen:[17,18],link:[0,1],list:[0,2,10,15,18],listcomp:[],liter:0,liu:9,live:15,llm_serv:15,lm:2,load:15,local:[2,5,10,15],local_devic:18,logic:[16,17],logical_mesh_shap:17,longer:10,look:[2,8],loop:18,loss:[0,17,18],loss_func:[0,17,18],lot:16,lower:[2,8,16,18],m:[2,8,10,17],machin:[2,17,18],made:2,mai:18,main:[1,16],mainli:[5,8],maintain:5,make:[8,10,15,16,17],manag:[2,8,10,17],mani:[0,2,8,16,17],manual:[0,15,16,18],manual_pipeline_actual_st:17,manual_pipeline_model:17,manual_pipeline_st:17,manual_pipeline_train_step:[0,17],manualpipelinemlpmodel:[0,17],manualstageopt:17,map:[0,2,17,18],mark:[0,17],mark_missing_vars_in_backward_computation_pipeline_mark:0,mark_pipelin:0,mark_pipeline_boundari:[0,16,17],marked_input:0,marked_output:0,marker:[0,17],materi:2,matmul:1,matrix:0,max_length:15,max_n_succ_stag:[],mb:19,md5:15,md:[4,5],mean:[0,10,16,17,18],meet:[8,10],megatron:2,mem:10,memori:[8,10,11,15,16],mention:10,merg:5,mesh:[0,16,17],mesh_shap:17,meshhostwork:[],meta:15,metadata:16,metaseq:15,method:[0,2,15,17],micro:[0,16,17],might:[0,2,8,10],mind:17,minim:[2,17],minimum:[8,10],minut:[10,11,17,18],ml:2,mlp:2,mlpmodel:[17,18],mlsy:9,mode:8,model:[0,2,4,7,8,9,16,17,18],model_nam:15,modif:5,modifi:[5,8,10],modul:[0,8,10,17,18],moduleprofileresult:17,mono:15,more:[0,2,8,11,16,17],most:[0,1,11,18],move:[0,8,16],ms:18,much:18,multipl:[0,2,9,10,17],multiplex:9,n:[2,10,17,18],name:[10,15],navig:1,nccl:8,nccl_error_unhandled_cuda_error:8,ncclerror:8,necessari:[2,5,17],need:[0,5,8,10,15,16,17,18],nest:0,network:[0,7,8,15,18],neural:[0,7,18],new_stat:[0,17,18],newer:8,next:[2,8,10,16],nice:16,nividia:8,nn:[0,17,18],node:[2,16,17],node_i:10,nodes_arrai:10,nontrivi:18,nor:15,normal:[17,18],note:[0,8,10,15,16,17],notebook:[11,14,17,18],noth:8,now:[0,2,8,10,17],np:18,npy:17,ntask:10,num:10,num_lay:18,num_micro_batch:[0,16,17],num_nod:17,number:[2,10,12,16,17,18],numpi:[15,17,18],nvcc:10,nvhpc:10,nvidia:[2,16],nvlink:2,obj_ref:[],object:[],obtain:[15,17],offer:15,offici:8,old:10,older:10,omegaconf:15,onc:[2,10],one:[2,8,10,15,17,18],one_replica_host_local_id:[],ones:[0,1,16],onli:[0,8,10,15,17,18],oom:15,op:[7,16,17],open:15,oper:[0,1,2,9,11,16,17,18],opt:7,opt_weight:15,optax:[8,17,18],optim:[0,2,9,16,17,18],option:[0,16,17],orchestr:2,order:[],org:[8,15],org_tensorflow:8,organ:0,origin:[0,2,8,15,18],orthogon:2,os:8,osdi:[2,9],other:[2,5,8,10],otherwis:8,our:[0,2,8],out:[0,8,10,15,17,18],output:[0,15,17,18],output_path:15,outvar_s:17,overal:0,overhead:2,overlap:16,override_repositori:8,overwrit:8,own:[],p3:[2,15],p:[9,10],pack:10,packag:[8,10,15],page:[1,8,9,10,15],pair:2,paper:[0,2,4,17],paral:17,paralel:18,parallel:[0,2,7,9,14,15,19],parallelizedfunc:0,param:[0,17,18],paramet:[0,4,11,17,18],paramt:[17,18],parcip:2,pari:15,part:[0,2,10],parti:5,partial:[2,18],particular:8,partit:[2,10,11,17],partition:0,partition_nam:10,pass:[0,8,16,17],path:[10,15],path_to_992_shard:15,path_to_save_checkpoint:15,pci:8,peak:15,peak_memori:17,per:[0,8,10,17,18],perform:[0,2,7,8,11,15],performac:15,peripher:8,phase:0,physic:17,pick:[11,15],pid:[],pip3:[8,15],pip:[5,8],pipelin:[0,2,7,11,14,16,18,19],pipeline_mark:0,pipeline_p:0,pipelineinstemitt:0,pipeshard_config:0,pipeshard_parallel:[17,19],pipeshardparallel:[0,16,17],pjit:[7,14,18,19],placement:[2,12],plan:[2,5,18],playground:5,pleas:[2,5,8,10,15],pmap:[7,14,19],pmap_lib:18,pmap_train_step:18,pmean:18,point:[1,8],port:[10,15],possibl:[1,15],post:2,potenti:16,power:15,pre:[10,15],prebuilt:8,precis:2,prefix:15,preprint:[],preserv:0,preshard_dynamic_arg:18,press:10,pretrain:15,prettier:1,previou:[10,17],primit:[0,16,17,18],primitive_def:0,print:[0,1,8,15,16,18],prngkei:[17,18],problem:[15,17,18],procedur:[8,15],process:[0,2,8,10,17],process_apply_gradi:0,profil:17,program:[2,8],project:[1,8,9,15,16],prompt:15,protocol:8,provid:[0,2,8,10,11,15,16,18],proxi:8,pseudo:0,pt:15,publicli:15,pull:5,purg:10,purpos:18,put:[10,12,15],pwd:8,py:[0,1,8,11,15,16,17,18,19],pylint:5,pypi:8,python3:[8,10,15],python:[0,1,5,10,11,12,14,15,17,18],pytorch:15,qirong:9,queri:15,question:7,queu:10,queue:10,quick:[16,17],quickstart:[7,14,19],ra:10,rai:[2,8,15],rais:[],ram:15,ran:13,random:[17,18],rang:[15,18],rational:2,raytaskerror:[],re:8,read:[2,8,10,15,17],readm:[4,5],real:10,realiz:2,rebas:8,rebuild:8,recent:[],recommend:[2,8],recompil:8,record:13,recurs:[5,8],redirect:8,reduc:18,reduct:8,ref:[],refer:[9,10],regardless:8,registernumpybfloat16:8,registr:1,regularli:8,reinstal:8,relat:0,reli:[8,16],relu:[0,17,18],remateri:0,remov:0,repeat:18,repetit:16,replac:16,replic:18,repo:[5,8,15],report:16,repositori:[5,8],repr:[],repres:[0,2],represent:[2,16,18],reproduc:[4,16],request:[5,8,15],requir:[0,2,8,11,16,17,18],research:9,reserach:13,reshap:18,resid:5,resourc:[8,15,17],respect:16,rest:0,result:[0,4,10,18],ret:[],return_tensor:15,review:5,rf:8,right:0,rm:8,rngkei:[17,18],roughli:10,rtol:[],run:[0,2,5,8,11,12,16,17,18],run_alpa_on_slurm:10,run_auto_sharding_pass:[0,1],runautoshardingpass:0,runtim:0,runtime_execut:18,runtimeerror:8,s:[0,2,5,8,10,15,16,17,18],sai:15,salesforc:15,same:[0,8,10,11,15,17,18],save:[15,17],scale:[4,7,18],scatter:1,schedul:[0,2,16],scheme:[2,18],scontrol:10,script:[5,11,12,15,16,17,18],second:[11,17,18],section:[4,8,10,15,17],secur:8,see:[0,5,8,10,12,15,16,17,18],self:[0,17,18],send:[2,5,16],sent:0,separ:[0,8,17],seper:[],sequenc:15,sequenti:0,serial:18,serial_execut:18,serv:[7,9],servic:[1,5,15],set:[0,1,2,8,10,16,17],setup:[8,15],sever:[2,11,16,17],sh:[5,10],shape:[16,18],shard:[2,7,14,15,16,19],shard_batch:18,shard_each_stag:0,shard_parallel:1,shardeddevicearrai:18,shardparallel:16,share:[8,10,15],shell:10,sheng:9,should:[0,5,8,10,12,15,16,17],show:[0,2,4,10,15,17,18],shown:[0,2],shutdown:17,side:1,signific:16,silent:8,similar:[8,15,18],similarli:10,simpl:[8,16,18],simpler:0,simpli:[0,15],sinc:8,singl:[0,2,11,15,16],site:13,size:[0,15,16,18],skip:15,skip_special_token:15,slack:15,sleep:10,slice:[0,2,16,17],slice_closed_jaxpr_by_full_pipeline_mark:0,sliceautoshardedstag:0,slide:[2,13],slow:15,slurm:7,slurm_cpus_per_task:10,slurm_job_nodelist:10,slurm_job_num_nod:10,small:16,smaller:[0,2,15],smi:16,snippet:0,so:[0,5,8,10,15,16,17,18],solv:[10,17,18],solver:[7,16],some:[0,2,8,10,12,15,16,17,18],some_jax_oper:0,sometim:[8,10],sourc:[5,11,14,15,17,18],space:15,spawn:[8,10],special:0,specif:[0,1,8,10,15,17,18],specifi:[0,8,10,15,17,18],sphinx:[11,14,17,18],split:[0,10,17,18],split_compute_grad_and_apply_grad:0,split_donate_invar:0,spmd:[0,1,5],squeue:10,srun:[8,10],stage:17,stage_opt:17,standard:8,star:7,start:[0,1,2,8,10,12,15,16,17],state:[0,16,17,18],statist:9,statu:[10,15],std:18,step:[0,2,5,8,10,15,17,18],step_2_consolidate_992_shards_to_singleton:15,step_3_convert_to_numpy_weight:15,still:18,stoica:9,stop:10,store:[5,16,18],strategi:[2,11,15,17,18],strictli:8,structur:[0,7],style:5,subgraph:2,subject:15,submesh:[0,17],submesh_choic:17,submit:[5,16],subset:[0,17],successfulli:15,suffici:15,suffix:8,suitabl:11,sum:0,summar:[0,10],summari:[10,11],support:[8,11,15,17],sure:[8,10,15,16],sync_func:18,synchron:16,synchronize_all_act:18,syntax:10,system:[2,7,8,10,15],t:[0,10],tabl:11,tag:8,tailor:18,take:[0,2,8,10,15,17,18],talk:2,target:[10,18],task:[0,8,10],team:13,techniqu:[2,11,18],tell:[10,15],temp_buffer_s:17,tensor:[2,16,18],tensorflow:[1,8],term:2,termin:[10,15],test:[1,3,10,16,17,18],test_bas:1,test_complet:15,test_debug_info:16,test_instal:[8,10],test_simpl:8,text:10,textgen:15,textgen_multinode_test:[],textgen_uninode_test:[],th:17,than:[8,10,11,18],thank:8,thei:[2,15,18],them:[0,2,10,11,17,18],therefor:[10,18],thi:[0,1,2,4,5,8,9,10,11,12,13,15,16,17,18],thin:0,thing:[10,18],think:16,third:[2,5],third_parti:[5,8],those:[2,8,16],thread:8,three:2,through:[2,7,8,10],ti:0,time:[8,10,11,17,18],tip:[15,16],todo:[2,11],tofu:2,togeth:10,toi:17,token:15,too:15,tool:[10,11,16],toolkit:8,top:[2,18],topic:13,topolog:10,torch:[8,15],torch_frontend:8,torchaudio:15,torchdistx:8,torchvis:15,total:[11,15,17,18,19],total_allocation_s:18,trace:0,traceback:[],traffic:8,train:[0,2,4,7,11,13,14,15,19],train_stat:[17,18],train_step:[17,18],trainstat:[17,18],tranform:18,transform:[0,11,15,18],tree:1,tree_map:18,tri:2,tricki:16,trigger:17,troubl:[8,15],truth:[17,18],tune:[7,15],tutori:[5,11,15,16,17,18,19],tutorials_jupyt:14,tutorials_python:14,twice:15,two:[0,2,8,11,15,17],tx:[17,18],type:[0,10,18],typic:[0,8],u:10,ubuntu:[],umath:8,under:[1,8,15,17],understand:[2,13,16,17],unexpect:16,unifi:2,uninstal:8,uniqu:15,unit:1,unlik:[2,18],up:[0,8,10],updat:[0,8],upgrad:8,upper:2,upstream:8,url:[8,15],us:[0,2,5,7,11,16,17,18],usag:[8,15,16],user:[0,2,8,10,15,16,17,18],usual:10,util:[15,17,18],uuid:[],uvicorn:15,v001:10,v0:1,v100:10,valu:[8,10,16],value_and_grad:16,variabl:[0,1,10,12,16],variant:2,variou:1,verfi:10,veri:13,verifi:15,version:[0,5,10,15],via:[0,5,8,15],view:[2,13,17],vincent:9,virtual:8,visual:[0,16],vt:8,vulner:8,w:[10,17,18],wai:[1,10,11],walk:[2,7],wang:9,want:[5,8,10,15,16,17,18],warmup:18,warn:16,we:[0,2,5,8,10,11,15,16,17,18],websit:15,weight:8,well:[2,10,18],what:2,wheel:15,when:[0,2,8,10,17],where:[0,1,10],whether:8,which:[0,2,5,10,11,13,15,17,18],whl:[8,15],whole:[0,17],whose:5,wise:16,within:[0,10],without:[8,10,15,18],work:[8,10,11,12,15,16,18],worker:[10,15,16,17],worker_num:10,workflow:[0,2,10],workload:11,wrapper:[0,15],write:[8,15,18],wrote:0,wx:[17,18],x:[0,2,15,17,18],xin:9,xing:9,xla:[0,1,2,5,8,18],xla_extens:18,xmap:18,xu:9,y:[0,8,17,18],yanp:9,yapf:5,ye:11,yida:9,ying:9,yinmin:9,yonghao:9,you:[0,1,2,5,8,10,12,15,16,17,18],your:[8,10,15,16,18],your_user_nam:10,yuanzhong:9,zero2parallel:16,zhang:9,zhao:9,zheng:9,zhifeng:9,zhong:9,zhuang:9,zhuo:9,zhuohan:9,zip:14},titles:["Alpa Compiler Walk-Through","Code Structure of the Intra-op Solver","Design and Architecture","Rationale","Performance Benchmark","Developer Guide","Alpa Tutorials","Alpa Documentation","Install Alpa","Publications","Using Alpa on Slurm","Differences between alpa.parallelize, jax.pmap and jax.pjit","Frequently Asked Questions (FAQ)","ICML\u201922 Big Model Tutorial","Alpa Tutorials","Serving OPT-175B, BLOOM-176B and CodeGen-16B using Alpa","Performance Tuning Guide","Distributed Training with Both Shard and Pipeline Parallelism","Alpa Quickstart","Computation times"],titleterms:{"1":[8,12],"16b":15,"175b":15,"176b":15,"2":[8,12],"22":13,"3":12,"import":[2,17,18],"public":[7,9],against:18,alpa:[0,5,6,7,8,10,11,12,14,15,18],an:[17,18],architectur:[2,7],argument:12,ask:12,assign:17,auto:[0,18],automat:17,benchmark:[4,7],between:11,big:13,bloom:15,both:17,check:[8,10],choos:16,cluster:[2,17],code:[1,5],codegen:15,command:15,compar:11,comparison:18,compil:[0,2],comput:[2,19],concept:2,connect:17,construct:0,contribut:5,control:12,convert:15,copi:15,creat:10,cuda:8,cuda_visible_devic:12,data:[11,18],debug:1,demo:15,depend:10,design:2,develop:[5,7],devic:[2,12,17,18],differ:11,distribut:17,document:7,enter:10,environ:[10,17],error:8,exampl:1,execut:18,exit:10,faq:12,featur:12,flax:8,format:[5,15],frequent:12,from:8,frontend:8,gener:15,get:7,gpu:[8,12],graph:2,guid:[5,7,16],hang:8,how:[1,12],icml:13,improv:15,init:12,initi:17,inspect:[1,16],instal:[8,10],inter:2,interact:10,interpret:17,intra:[1,2],jax:[8,11,18],jaxlib:8,job:10,kei:1,launch:15,layer:0,learn:1,librari:[17,18],licens:15,line:15,lint:5,load:10,manual:17,memori:18,mesh:2,method:[8,12,16],mlp:[17,18],mode:10,model:[11,13,15],multi:[8,10],multipl:15,node:[8,10,15],numpi:8,op:[1,2],opt:15,option:[8,10],orchestratoin:2,organ:5,other:[12,15],output:10,overhead:16,overview:[2,15],parallel:[11,16,17,18],pass:2,perform:[4,16],pipelin:17,pjit:11,pmap:[11,18],pointer:1,prefer:11,prerequisit:[8,10],problem:8,python:8,pytorch:8,question:12,quickstart:18,rai:[10,12,17],rational:3,read:1,reduc:16,requir:[10,15],reshard:2,resourc:10,result:17,run:[1,10,15],runtim:[2,16],sampl:10,sbatch:10,script:10,serv:15,server:15,session:10,setup:10,shard:[0,1,17],simpl:1,singl:[10,17,18],slurm:[8,10],softwar:10,solver:1,some:1,sourc:8,speed:[15,18],stage:[0,2],start:7,startup:10,strategi:[1,16],structur:1,submit:10,submodul:5,tensorflow:5,test:[5,8],through:0,time:19,train:[17,18],troubleshoot:8,tune:16,tutori:[6,7,13,14],unhandl:8,unit:5,updat:5,us:[8,10,12,15],usag:18,version:8,virtual:10,walk:0,web:15,weight:15,wheel:8,when:11,worker:2,xmap:11}})