Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
!==============================================================================!
! |
! ||===\\ |
! || \\ |
! || || //==\\ || || //==|| ||/==\\ |
! || || || || || || || || || || |
! || // || || || || || || || |
! ||===// \\==// \\==\\ \\==\\ || |
! |
!==============================================================================!
!==============================================================================!
! |
! WSMP_SETUP1 Oct. 2008 |
! WSMP_SETUP2 Oct. 2008 |
! |
!==============================================================================!
!==============================================================================!
subroutine wsmp_setup1 (n,n_iproc_st,n_iproc_end,n_iproc,nz_loc,ldb,ndof,vo,osolve,tpl,params,threadinfo,istep,iter)
use definitions
use threads
implicit none
integer n,n_iproc_st,n_iproc_end,n_iproc,nz_loc,ldb
integer ndof
type (void) vo
type (topology) :: tpl(n)
type (octreesolve) osolve
type (parameters) params
type (thread) threadinfo
integer istep,iter
logical, dimension(:), allocatable :: iproc_col
integer icol(100000),kk,nn,n0,jproc,idg_locp,nz
integer ie,k1,k2,j,i2,i1,idof1,idof2,k,err,i3,k3,i
integer iproc,nproc,ierr
character*72 :: shift
integer, dimension(:), allocatable :: irn,jcn
INCLUDE 'mpif.h'
call mpi_comm_size (mpi_comm_world,nproc,ierr)
call mpi_comm_rank (mpi_comm_world,iproc,ierr)
shift=' '
!==============================================================================!
!==============================================================================!
! n is the size of the system/matrix
! nz is the total number of non-zero terms in the global matrix
! n_iproc is the number of columns on given thread iproc
! n_iproc_st is the first column on given thread iproc, between 1 and n
! n_iproc_end is the last column on given thread iproc, between 1 and n
! nz_loc is the total number of non-zero terms on given thread iproc
! ia is an integer array of size n_iproc+1 (see wsmp manual)
! ja is an interer array of size nz_loc
! avals is a double precision array of size nz_loc containing the values
! b is a double precision array of size n_iproc containing the rhs
!==============================================================================!
!=====[allocate memory for topology]===========================================!
!==============================================================================!
tpl%nheight=0
do i=1,n
tpl(i)%icol(:)=0
enddo
!call heap (threadinfo,'tpl(:)%icol(:)','wsmp_setup',wsmp%n*tpl%nheightmax,'int',+1)
!==============================================================================!
!=====[find_connectivity_dimension]============================================!
!==============================================================================!
! the list of non-zero terms is computed for every column
! i1 is a column index
! i2 is a row index
! for symmetric systems, we only build and store the lower triangular part
! of the matrix, i.e. i2.ge.i1
do ie=1,osolve%nleaves
if (vo%leaf(ie).eq.0) then
do k1=1,params%mpe
do idof1=1,ndof
i1=(vo%ftr(osolve%icon(k1,ie))-1)*ndof+idof1
do k2=1,params%mpe
do idof2=1,ndof
i2=(vo%ftr(osolve%icon(k2,ie))-1)*ndof+idof2
if (i1.gt.i2.and.ndof.gt.1) goto 2
do j=1,tpl(i1)%nheight
if (i2.eq.tpl(i1)%icol(j)) goto 2
enddo
if (tpl(i1)%nheight.eq.tpl(i1)%nheightmax) then
do k=1,tpl(i1)%nheight
icol(k)=tpl(i1)%icol(k)
enddo
if (tpl(i1)%nheight.gt.0) deallocate (tpl(i1)%icol)
tpl(i1)%nheightmax=tpl(i1)%nheightmax+ndof
allocate (tpl(i1)%icol(tpl(i1)%nheightmax),stat=err)
tpl(i1)%icol(:)=0
if (err.ne.0) call stop_run ('Error alloc tpl(i1)%icol$')
do k=1,tpl(i1)%nheight
tpl(i1)%icol(k)=icol(k)
enddo
endif
tpl(i1)%nheight=tpl(i1)%nheight+1
tpl(i1)%icol(tpl(i1)%nheight)=i2
2 continue
enddo
enddo
enddo
enddo
endif
enddo
!print *,iproc,'nface=',osolve%nface
do ie=1,osolve%nface
if (vo%face(ie).eq.0) then
do k1=1,9
do idof1=1,ndof
i1=(vo%ftr(osolve%iface(k1,ie))-1)*ndof+idof1
do k2=1,9
do idof2=1,ndof
i2=(vo%ftr(osolve%iface(k2,ie))-1)*ndof+idof2
if (i1.gt.i2.and.ndof.gt.1) goto 3 !!!!!!NEW
do j=1,tpl(i1)%nheight
if (i2.eq.tpl(i1)%icol(j)) goto 3
enddo
if (tpl(i1)%nheight.eq.tpl(i1)%nheightmax) then
do k=1,tpl(i1)%nheight
icol(k)=tpl(i1)%icol(k)
enddo
if (tpl(i1)%nheight.gt.0) deallocate (tpl(i1)%icol)
tpl(i1)%nheightmax=tpl(i1)%nheightmax+ndof
allocate (tpl(i1)%icol(tpl(i1)%nheightmax),stat=err)
if (err.ne.0) call stop_run ('Error alloc tpl(i1)%icol$')
do k=1,tpl(i1)%nheight
tpl(i1)%icol(k)=icol(k)
enddo
endif
tpl(i1)%nheight=tpl(i1)%nheight+1
tpl(i1)%icol(tpl(i1)%nheight)=i2
3 continue
enddo
enddo
enddo
enddo
endif
enddo
!print *,iproc,'min/max tpl%nheight',minval(tpl(:)%nheight),maxval(tpl(:)%nheight)
!print *,iproc,'avrg tpl%nheight',sum(tpl(:)%nheight)/dble(wsmp%n)
!==============================================================================!
!=====[computing total number of non-zero terms]===============================!
!==============================================================================!
! the total number of non-zero terms is the sum of the height of each column
nz=sum(tpl(1:vo%nnode*ndof)%nheight)
if (iproc.eq.0) write(*,'(a,i9)') shift//'n =',n
if (iproc.eq.0) write(*,'(a,i9)') shift//'nz=',nz
!==============================================================================!
!=====[sorting icol's]=========================================================!
!==============================================================================!
! wsmp requires the indices of the row contained in each column to be sorted
do i=1,n
call iqsort_s (tpl(i)%icol,tpl(i)%nheight)
end do
!==============================================================================!
!=====[find_processors]========================================================!
!==============================================================================!
nn =(n-1)/nproc+1
n_iproc_st = nn*iproc+1
n_iproc_end = min(n_iproc_st+nn-1,n)
n_iproc = n_iproc_end - n_iproc_st +1
ldb = n_iproc
allocate(iproc_col(n))
iproc_col=.false.
iproc_col(n_iproc_st:n_iproc_end)=.true.
nz_loc=sum(tpl(:)%nheight,iproc_col)
deallocate(iproc_col)
!==============================================================================!
!=====[outputs visual reprenstation of matrix]=================================!
!==============================================================================!
if (params%visualise_matrix) then
allocate(irn(nz)) ; call heap (threadinfo,'irn','wsmp_setup',size(irn),'int',+1)
allocate(jcn(nz)) ; call heap (threadinfo,'jcn','wsmp_setup',size(jcn),'int',+1)
irn=0
jcn=0
! the coordinates (irn,jcn) of the non zero terms in the global matrix are
! computed, as well as the indices of the diagonal terms in the list of dof's.
kk=0
do i=1,n
do j=1,tpl(i)%nheight
kk=kk+1
if (kk.gt.nz) call stop_run ('error in find_connectivity$')
irn(kk)=tpl(i)%icol(j)
jcn(kk)=i
enddo
enddo
! call visualise_matrix (nz,irn,jcn,n,istep,iter,ndof)
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
call heap (threadinfo,'irn','wsmp_setup',size(irn),'int',-1) ; deallocate (irn)
call heap (threadinfo,'jcn','wsmp_setup',size(jcn),'int',-1) ; deallocate (jcn)
end if
end subroutine wsmp_setup1
!==============================================================================!
!==============================================================================!
!
! WSMP_SETUP2 Oct. 2008 |
!
!==============================================================================!
!==============================================================================!
subroutine wsmp_setup2 (n,n_iproc,n_iproc_st,n_iproc_end,nz_loc,iproc_col, &
ia,ja,ndof,vo,osolve,tpl,params,threadinfo,istep,iter)
use definitions
use threads
implicit none
integer n,nz_loc,n_iproc,n_iproc_st,n_iproc_end
logical iproc_col(n)
integer ia(n_iproc+1)
integer ja(nz_loc)
integer ndof
type (void) vo
type (topology) :: tpl(n)
type (octreesolve) osolve
type (parameters) params
type (thread) threadinfo
integer istep,iter
integer iloc,iatemp,nheightp,i,k
integer iproc,nproc,ierr
INCLUDE 'mpif.h'
call mpi_comm_size (mpi_comm_world,nproc,ierr)
call mpi_comm_rank (mpi_comm_world,iproc,ierr)
iproc_col=.false.
iproc_col(n_iproc_st:n_iproc_end)=.true.
!==============================================================================!
!=====[find_connectivity_local]================================================!
!==============================================================================!
iloc=1
do i=1,n
if (iproc_col(i)) then
do k=1,tpl(i)%nheight
ja(iloc+k-1)=tpl(i)%icol(k)
enddo
iloc=iloc+tpl(i)%nheight
endif
enddo
!==============================================================================!
!=====[build ia,ja]============================================================!
!==============================================================================!
iatemp=0
k=0
do i=n_iproc_st,n_iproc_end
k=k+1
if (iatemp==0) then
ia(k)=1
else
ia(k)=iatemp+nheightp
end if
iatemp=ia(k)
nheightp=tpl(i)%nheight
end do
k=k+1
ia(k)=iatemp+nheightp
end subroutine wsmp_setup2
!==============================================================================!
!==============================================================================!