File: | home/bhubbard/working/src/ceph/src/spdk/dpdk/lib/librte_eal/linux/eal/eal_memory.c |
Warning: | line 2308, column 21 Division by zero |
[?] Use j/k keys for keyboard navigation
1 | /* SPDX-License-Identifier: BSD-3-Clause | |||||
2 | * Copyright(c) 2010-2014 Intel Corporation. | |||||
3 | * Copyright(c) 2013 6WIND S.A. | |||||
4 | */ | |||||
5 | ||||||
6 | #define _FILE_OFFSET_BITS64 64 | |||||
7 | #include <errno(*__errno_location ()).h> | |||||
8 | #include <fcntl.h> | |||||
9 | #include <stdarg.h> | |||||
10 | #include <stdbool.h> | |||||
11 | #include <stdlib.h> | |||||
12 | #include <stdio.h> | |||||
13 | #include <stdint.h> | |||||
14 | #include <inttypes.h> | |||||
15 | #include <string.h> | |||||
16 | #include <sys/mman.h> | |||||
17 | #include <sys/types.h> | |||||
18 | #include <sys/stat.h> | |||||
19 | #include <sys/queue.h> | |||||
20 | #include <sys/file.h> | |||||
21 | #include <sys/resource.h> | |||||
22 | #include <unistd.h> | |||||
23 | #include <limits.h> | |||||
24 | #include <sys/ioctl.h> | |||||
25 | #include <sys/time.h> | |||||
26 | #include <signal.h> | |||||
27 | #include <setjmp.h> | |||||
28 | #ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */ | |||||
29 | #include <linux1/memfd.h> | |||||
30 | #define MEMFD_SUPPORTED | |||||
31 | #endif | |||||
32 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
33 | #include <numa.h> | |||||
34 | #include <numaif.h> | |||||
35 | #endif | |||||
36 | ||||||
37 | #include <rte_errno(per_lcore__rte_errno).h> | |||||
38 | #include <rte_log.h> | |||||
39 | #include <rte_memory.h> | |||||
40 | #include <rte_launch.h> | |||||
41 | #include <rte_eal.h> | |||||
42 | #include <rte_eal_memconfig.h> | |||||
43 | #include <rte_per_lcore.h> | |||||
44 | #include <rte_lcore.h> | |||||
45 | #include <rte_common.h> | |||||
46 | #include <rte_string_fns.h> | |||||
47 | ||||||
48 | #include "eal_private.h" | |||||
49 | #include "eal_memalloc.h" | |||||
50 | #include "eal_internal_cfg.h" | |||||
51 | #include "eal_filesystem.h" | |||||
52 | #include "eal_hugepages.h" | |||||
53 | #include "eal_options.h" | |||||
54 | ||||||
55 | #define PFN_MASK_SIZE8 8 | |||||
56 | ||||||
57 | /** | |||||
58 | * @file | |||||
59 | * Huge page mapping under linux | |||||
60 | * | |||||
61 | * To reserve a big contiguous amount of memory, we use the hugepage | |||||
62 | * feature of linux. For that, we need to have hugetlbfs mounted. This | |||||
63 | * code will create many files in this directory (one per page) and | |||||
64 | * map them in virtual memory. For each page, we will retrieve its | |||||
65 | * physical address and remap it in order to have a virtual contiguous | |||||
66 | * zone as well as a physical contiguous zone. | |||||
67 | */ | |||||
68 | ||||||
69 | static bool_Bool phys_addrs_available = true1; | |||||
70 | ||||||
71 | #define RANDOMIZE_VA_SPACE_FILE"/proc/sys/kernel/randomize_va_space" "/proc/sys/kernel/randomize_va_space" | |||||
72 | ||||||
73 | static void | |||||
74 | test_phys_addrs_available(void) | |||||
75 | { | |||||
76 | uint64_t tmp = 0; | |||||
77 | phys_addr_t physaddr; | |||||
78 | ||||||
79 | if (!rte_eal_has_hugepages()) { | |||||
80 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "Started without hugepages support, physical addresses not available\n" ) | |||||
81 | "Started without hugepages support, physical addresses not available\n")rte_log(4U, 0, "EAL" ": " "Started without hugepages support, physical addresses not available\n" ); | |||||
82 | phys_addrs_available = false0; | |||||
83 | return; | |||||
84 | } | |||||
85 | ||||||
86 | physaddr = rte_mem_virt2phy(&tmp); | |||||
87 | if (physaddr == RTE_BAD_PHYS_ADDR((phys_addr_t)-1)) { | |||||
88 | if (rte_eal_iova_mode() == RTE_IOVA_PA) | |||||
89 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. " "Only vfio will function.\n", strerror((*__errno_location () ))) | |||||
90 | "Cannot obtain physical addresses: %s. "rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. " "Only vfio will function.\n", strerror((*__errno_location () ))) | |||||
91 | "Only vfio will function.\n",rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. " "Only vfio will function.\n", strerror((*__errno_location () ))) | |||||
92 | strerror(errno))rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. " "Only vfio will function.\n", strerror((*__errno_location () ))); | |||||
93 | phys_addrs_available = false0; | |||||
94 | } | |||||
95 | } | |||||
96 | ||||||
97 | /* | |||||
98 | * Get physical address of any mapped virtual address in the current process. | |||||
99 | */ | |||||
100 | phys_addr_t | |||||
101 | rte_mem_virt2phy(const void *virtaddr) | |||||
102 | { | |||||
103 | int fd, retval; | |||||
104 | uint64_t page, physaddr; | |||||
105 | unsigned long virt_pfn; | |||||
106 | int page_size; | |||||
107 | off_t offset; | |||||
108 | ||||||
109 | /* Cannot parse /proc/self/pagemap, no need to log errors everywhere */ | |||||
110 | if (!phys_addrs_available) | |||||
111 | return RTE_BAD_IOVA((rte_iova_t)-1); | |||||
112 | ||||||
113 | /* standard page size */ | |||||
114 | page_size = getpagesize(); | |||||
115 | ||||||
116 | fd = open("/proc/self/pagemap", O_RDONLY00); | |||||
117 | if (fd < 0) { | |||||
118 | RTE_LOG(INFO, EAL, "%s(): cannot open /proc/self/pagemap: %s\n",rte_log(7U, 0, "EAL" ": " "%s(): cannot open /proc/self/pagemap: %s\n" , __func__, strerror((*__errno_location ()))) | |||||
119 | __func__, strerror(errno))rte_log(7U, 0, "EAL" ": " "%s(): cannot open /proc/self/pagemap: %s\n" , __func__, strerror((*__errno_location ()))); | |||||
120 | return RTE_BAD_IOVA((rte_iova_t)-1); | |||||
121 | } | |||||
122 | ||||||
123 | virt_pfn = (unsigned long)virtaddr / page_size; | |||||
124 | offset = sizeof(uint64_t) * virt_pfn; | |||||
125 | if (lseek(fd, offset, SEEK_SET0) == (off_t) -1) { | |||||
126 | RTE_LOG(INFO, EAL, "%s(): seek error in /proc/self/pagemap: %s\n",rte_log(7U, 0, "EAL" ": " "%s(): seek error in /proc/self/pagemap: %s\n" , __func__, strerror((*__errno_location ()))) | |||||
127 | __func__, strerror(errno))rte_log(7U, 0, "EAL" ": " "%s(): seek error in /proc/self/pagemap: %s\n" , __func__, strerror((*__errno_location ()))); | |||||
128 | close(fd); | |||||
129 | return RTE_BAD_IOVA((rte_iova_t)-1); | |||||
130 | } | |||||
131 | ||||||
132 | retval = read(fd, &page, PFN_MASK_SIZE8); | |||||
133 | close(fd); | |||||
134 | if (retval < 0) { | |||||
135 | RTE_LOG(INFO, EAL, "%s(): cannot read /proc/self/pagemap: %s\n",rte_log(7U, 0, "EAL" ": " "%s(): cannot read /proc/self/pagemap: %s\n" , __func__, strerror((*__errno_location ()))) | |||||
136 | __func__, strerror(errno))rte_log(7U, 0, "EAL" ": " "%s(): cannot read /proc/self/pagemap: %s\n" , __func__, strerror((*__errno_location ()))); | |||||
137 | return RTE_BAD_IOVA((rte_iova_t)-1); | |||||
138 | } else if (retval != PFN_MASK_SIZE8) { | |||||
139 | RTE_LOG(INFO, EAL, "%s(): read %d bytes from /proc/self/pagemap "rte_log(7U, 0, "EAL" ": " "%s(): read %d bytes from /proc/self/pagemap " "but expected %d:\n", __func__, retval, 8) | |||||
140 | "but expected %d:\n",rte_log(7U, 0, "EAL" ": " "%s(): read %d bytes from /proc/self/pagemap " "but expected %d:\n", __func__, retval, 8) | |||||
141 | __func__, retval, PFN_MASK_SIZE)rte_log(7U, 0, "EAL" ": " "%s(): read %d bytes from /proc/self/pagemap " "but expected %d:\n", __func__, retval, 8); | |||||
142 | return RTE_BAD_IOVA((rte_iova_t)-1); | |||||
143 | } | |||||
144 | ||||||
145 | /* | |||||
146 | * the pfn (page frame number) are bits 0-54 (see | |||||
147 | * pagemap.txt in linux Documentation) | |||||
148 | */ | |||||
149 | if ((page & 0x7fffffffffffffULL) == 0) | |||||
150 | return RTE_BAD_IOVA((rte_iova_t)-1); | |||||
151 | ||||||
152 | physaddr = ((page & 0x7fffffffffffffULL) * page_size) | |||||
153 | + ((unsigned long)virtaddr % page_size); | |||||
154 | ||||||
155 | return physaddr; | |||||
156 | } | |||||
157 | ||||||
158 | rte_iova_t | |||||
159 | rte_mem_virt2iova(const void *virtaddr) | |||||
160 | { | |||||
161 | if (rte_eal_iova_mode() == RTE_IOVA_VA) | |||||
162 | return (uintptr_t)virtaddr; | |||||
163 | return rte_mem_virt2phy(virtaddr); | |||||
164 | } | |||||
165 | ||||||
166 | /* | |||||
167 | * For each hugepage in hugepg_tbl, fill the physaddr value. We find | |||||
168 | * it by browsing the /proc/self/pagemap special file. | |||||
169 | */ | |||||
170 | static int | |||||
171 | find_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) | |||||
172 | { | |||||
173 | unsigned int i; | |||||
174 | phys_addr_t addr; | |||||
175 | ||||||
176 | for (i = 0; i < hpi->num_pages[0]; i++) { | |||||
177 | addr = rte_mem_virt2phy(hugepg_tbl[i].orig_va); | |||||
178 | if (addr == RTE_BAD_PHYS_ADDR((phys_addr_t)-1)) | |||||
179 | return -1; | |||||
180 | hugepg_tbl[i].physaddr = addr; | |||||
181 | } | |||||
182 | return 0; | |||||
183 | } | |||||
184 | ||||||
185 | /* | |||||
186 | * For each hugepage in hugepg_tbl, fill the physaddr value sequentially. | |||||
187 | */ | |||||
188 | static int | |||||
189 | set_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) | |||||
190 | { | |||||
191 | unsigned int i; | |||||
192 | static phys_addr_t addr; | |||||
193 | ||||||
194 | for (i = 0; i < hpi->num_pages[0]; i++) { | |||||
195 | hugepg_tbl[i].physaddr = addr; | |||||
196 | addr += hugepg_tbl[i].size; | |||||
197 | } | |||||
198 | return 0; | |||||
199 | } | |||||
200 | ||||||
201 | /* | |||||
202 | * Check whether address-space layout randomization is enabled in | |||||
203 | * the kernel. This is important for multi-process as it can prevent | |||||
204 | * two processes mapping data to the same virtual address | |||||
205 | * Returns: | |||||
206 | * 0 - address space randomization disabled | |||||
207 | * 1/2 - address space randomization enabled | |||||
208 | * negative error code on error | |||||
209 | */ | |||||
210 | static int | |||||
211 | aslr_enabled(void) | |||||
212 | { | |||||
213 | char c; | |||||
214 | int retval, fd = open(RANDOMIZE_VA_SPACE_FILE"/proc/sys/kernel/randomize_va_space", O_RDONLY00); | |||||
215 | if (fd < 0) | |||||
216 | return -errno(*__errno_location ()); | |||||
217 | retval = read(fd, &c, 1); | |||||
218 | close(fd); | |||||
219 | if (retval < 0) | |||||
220 | return -errno(*__errno_location ()); | |||||
221 | if (retval == 0) | |||||
222 | return -EIO5; | |||||
223 | switch (c) { | |||||
224 | case '0' : return 0; | |||||
225 | case '1' : return 1; | |||||
226 | case '2' : return 2; | |||||
227 | default: return -EINVAL22; | |||||
228 | } | |||||
229 | } | |||||
230 | ||||||
231 | static sigjmp_buf huge_jmpenv; | |||||
232 | ||||||
233 | static void huge_sigbus_handler(int signo __rte_unused__attribute__((__unused__))) | |||||
234 | { | |||||
235 | siglongjmp(huge_jmpenv, 1); | |||||
236 | } | |||||
237 | ||||||
238 | /* Put setjmp into a wrap method to avoid compiling error. Any non-volatile, | |||||
239 | * non-static local variable in the stack frame calling sigsetjmp might be | |||||
240 | * clobbered by a call to longjmp. | |||||
241 | */ | |||||
242 | static int huge_wrap_sigsetjmp(void) | |||||
243 | { | |||||
244 | return sigsetjmp(huge_jmpenv, 1)__sigsetjmp (huge_jmpenv, 1); | |||||
245 | } | |||||
246 | ||||||
247 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
248 | /* Callback for numa library. */ | |||||
249 | void numa_error(char *where) | |||||
250 | { | |||||
251 | RTE_LOG(ERR, EAL, "%s failed: %s\n", where, strerror(errno))rte_log(4U, 0, "EAL" ": " "%s failed: %s\n", where, strerror( (*__errno_location ()))); | |||||
252 | } | |||||
253 | #endif | |||||
254 | ||||||
255 | /* | |||||
256 | * Mmap all hugepages of hugepage table: it first open a file in | |||||
257 | * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the | |||||
258 | * virtual address is stored in hugepg_tbl[i].orig_va, else it is stored | |||||
259 | * in hugepg_tbl[i].final_va. The second mapping (when orig is 0) tries to | |||||
260 | * map contiguous physical blocks in contiguous virtual blocks. | |||||
261 | */ | |||||
262 | static unsigned | |||||
263 | map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi, | |||||
264 | uint64_t *essential_memory __rte_unused__attribute__((__unused__))) | |||||
265 | { | |||||
266 | int fd; | |||||
267 | unsigned i; | |||||
268 | void *virtaddr; | |||||
269 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
270 | int node_id = -1; | |||||
271 | int essential_prev = 0; | |||||
272 | int oldpolicy; | |||||
273 | struct bitmask *oldmask = NULL((void*)0); | |||||
274 | bool_Bool have_numa = true1; | |||||
275 | unsigned long maxnode = 0; | |||||
276 | ||||||
277 | /* Check if kernel supports NUMA. */ | |||||
278 | if (numa_available() != 0) { | |||||
279 | RTE_LOG(DEBUG, EAL, "NUMA is not supported.\n")rte_log(8U, 0, "EAL" ": " "NUMA is not supported.\n"); | |||||
280 | have_numa = false0; | |||||
281 | } | |||||
282 | ||||||
283 | if (have_numa) { | |||||
284 | RTE_LOG(DEBUG, EAL, "Trying to obtain current memory policy.\n")rte_log(8U, 0, "EAL" ": " "Trying to obtain current memory policy.\n" ); | |||||
285 | oldmask = numa_allocate_nodemask(); | |||||
286 | if (get_mempolicy(&oldpolicy, oldmask->maskp, | |||||
287 | oldmask->size + 1, 0, 0) < 0) { | |||||
288 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "Failed to get current mempolicy: %s. " "Assuming MPOL_DEFAULT.\n", strerror((*__errno_location ())) ) | |||||
289 | "Failed to get current mempolicy: %s. "rte_log(4U, 0, "EAL" ": " "Failed to get current mempolicy: %s. " "Assuming MPOL_DEFAULT.\n", strerror((*__errno_location ())) ) | |||||
290 | "Assuming MPOL_DEFAULT.\n", strerror(errno))rte_log(4U, 0, "EAL" ": " "Failed to get current mempolicy: %s. " "Assuming MPOL_DEFAULT.\n", strerror((*__errno_location ())) ); | |||||
291 | oldpolicy = MPOL_DEFAULT; | |||||
292 | } | |||||
293 | for (i = 0; i < RTE_MAX_NUMA_NODES8; i++) | |||||
294 | if (internal_config.socket_mem[i]) | |||||
295 | maxnode = i + 1; | |||||
296 | } | |||||
297 | #endif | |||||
298 | ||||||
299 | for (i = 0; i < hpi->num_pages[0]; i++) { | |||||
300 | struct hugepage_file *hf = &hugepg_tbl[i]; | |||||
301 | uint64_t hugepage_sz = hpi->hugepage_sz; | |||||
302 | ||||||
303 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
304 | if (maxnode) { | |||||
305 | unsigned int j; | |||||
306 | ||||||
307 | for (j = 0; j < maxnode; j++) | |||||
308 | if (essential_memory[j]) | |||||
309 | break; | |||||
310 | ||||||
311 | if (j == maxnode) { | |||||
312 | node_id = (node_id + 1) % maxnode; | |||||
313 | while (!internal_config.socket_mem[node_id]) { | |||||
314 | node_id++; | |||||
315 | node_id %= maxnode; | |||||
316 | } | |||||
317 | essential_prev = 0; | |||||
318 | } else { | |||||
319 | node_id = j; | |||||
320 | essential_prev = essential_memory[j]; | |||||
321 | ||||||
322 | if (essential_memory[j] < hugepage_sz) | |||||
323 | essential_memory[j] = 0; | |||||
324 | else | |||||
325 | essential_memory[j] -= hugepage_sz; | |||||
326 | } | |||||
327 | ||||||
328 | RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Setting policy MPOL_PREFERRED for socket %d\n" , node_id) | |||||
329 | "Setting policy MPOL_PREFERRED for socket %d\n",rte_log(8U, 0, "EAL" ": " "Setting policy MPOL_PREFERRED for socket %d\n" , node_id) | |||||
330 | node_id)rte_log(8U, 0, "EAL" ": " "Setting policy MPOL_PREFERRED for socket %d\n" , node_id); | |||||
331 | numa_set_preferred(node_id); | |||||
332 | } | |||||
333 | #endif | |||||
334 | ||||||
335 | hf->file_id = i; | |||||
336 | hf->size = hugepage_sz; | |||||
337 | eal_get_hugefile_path(hf->filepath, sizeof(hf->filepath), | |||||
338 | hpi->hugedir, hf->file_id); | |||||
339 | hf->filepath[sizeof(hf->filepath) - 1] = '\0'; | |||||
340 | ||||||
341 | /* try to create hugepage file */ | |||||
342 | fd = open(hf->filepath, O_CREAT0100 | O_RDWR02, 0600); | |||||
343 | if (fd < 0) { | |||||
344 | RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__,rte_log(8U, 0, "EAL" ": " "%s(): open failed: %s\n", __func__ , strerror((*__errno_location ()))) | |||||
345 | strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): open failed: %s\n", __func__ , strerror((*__errno_location ()))); | |||||
346 | goto out; | |||||
347 | } | |||||
348 | ||||||
349 | /* map the segment, and populate page tables, | |||||
350 | * the kernel fills this segment with zeros. we don't care where | |||||
351 | * this gets mapped - we already have contiguous memory areas | |||||
352 | * ready for us to map into. | |||||
353 | */ | |||||
354 | virtaddr = mmap(NULL((void*)0), hugepage_sz, PROT_READ0x1 | PROT_WRITE0x2, | |||||
355 | MAP_SHARED0x01 | MAP_POPULATE0x08000, fd, 0); | |||||
356 | if (virtaddr == MAP_FAILED((void *) -1)) { | |||||
357 | RTE_LOG(DEBUG, EAL, "%s(): mmap failed: %s\n", __func__,rte_log(8U, 0, "EAL" ": " "%s(): mmap failed: %s\n", __func__ , strerror((*__errno_location ()))) | |||||
358 | strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): mmap failed: %s\n", __func__ , strerror((*__errno_location ()))); | |||||
359 | close(fd); | |||||
360 | goto out; | |||||
361 | } | |||||
362 | ||||||
363 | hf->orig_va = virtaddr; | |||||
364 | ||||||
365 | /* In linux, hugetlb limitations, like cgroup, are | |||||
366 | * enforced at fault time instead of mmap(), even | |||||
367 | * with the option of MAP_POPULATE. Kernel will send | |||||
368 | * a SIGBUS signal. To avoid to be killed, save stack | |||||
369 | * environment here, if SIGBUS happens, we can jump | |||||
370 | * back here. | |||||
371 | */ | |||||
372 | if (huge_wrap_sigsetjmp()) { | |||||
373 | RTE_LOG(DEBUG, EAL, "SIGBUS: Cannot mmap more "rte_log(8U, 0, "EAL" ": " "SIGBUS: Cannot mmap more " "hugepages of size %u MB\n" , (unsigned int)(hugepage_sz / 0x100000)) | |||||
374 | "hugepages of size %u MB\n",rte_log(8U, 0, "EAL" ": " "SIGBUS: Cannot mmap more " "hugepages of size %u MB\n" , (unsigned int)(hugepage_sz / 0x100000)) | |||||
375 | (unsigned int)(hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "SIGBUS: Cannot mmap more " "hugepages of size %u MB\n" , (unsigned int)(hugepage_sz / 0x100000)); | |||||
376 | munmap(virtaddr, hugepage_sz); | |||||
377 | close(fd); | |||||
378 | unlink(hugepg_tbl[i].filepath); | |||||
379 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
380 | if (maxnode) | |||||
381 | essential_memory[node_id] = | |||||
382 | essential_prev; | |||||
383 | #endif | |||||
384 | goto out; | |||||
385 | } | |||||
386 | *(int *)virtaddr = 0; | |||||
387 | ||||||
388 | /* set shared lock on the file. */ | |||||
389 | if (flock(fd, LOCK_SH1) < 0) { | |||||
390 | RTE_LOG(DEBUG, EAL, "%s(): Locking file failed:%s \n",rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed:%s \n", __func__ , strerror((*__errno_location ()))) | |||||
391 | __func__, strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed:%s \n", __func__ , strerror((*__errno_location ()))); | |||||
392 | close(fd); | |||||
393 | goto out; | |||||
394 | } | |||||
395 | ||||||
396 | close(fd); | |||||
397 | } | |||||
398 | ||||||
399 | out: | |||||
400 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
401 | if (maxnode) { | |||||
402 | RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Restoring previous memory policy: %d\n" , oldpolicy) | |||||
403 | "Restoring previous memory policy: %d\n", oldpolicy)rte_log(8U, 0, "EAL" ": " "Restoring previous memory policy: %d\n" , oldpolicy); | |||||
404 | if (oldpolicy == MPOL_DEFAULT) { | |||||
405 | numa_set_localalloc(); | |||||
406 | } else if (set_mempolicy(oldpolicy, oldmask->maskp, | |||||
407 | oldmask->size + 1) < 0) { | |||||
408 | RTE_LOG(ERR, EAL, "Failed to restore mempolicy: %s\n",rte_log(4U, 0, "EAL" ": " "Failed to restore mempolicy: %s\n" , strerror((*__errno_location ()))) | |||||
409 | strerror(errno))rte_log(4U, 0, "EAL" ": " "Failed to restore mempolicy: %s\n" , strerror((*__errno_location ()))); | |||||
410 | numa_set_localalloc(); | |||||
411 | } | |||||
412 | } | |||||
413 | if (oldmask != NULL((void*)0)) | |||||
414 | numa_free_cpumask(oldmask); | |||||
415 | #endif | |||||
416 | return i; | |||||
417 | } | |||||
418 | ||||||
419 | /* | |||||
420 | * Parse /proc/self/numa_maps to get the NUMA socket ID for each huge | |||||
421 | * page. | |||||
422 | */ | |||||
423 | static int | |||||
424 | find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) | |||||
425 | { | |||||
426 | int socket_id; | |||||
427 | char *end, *nodestr; | |||||
428 | unsigned i, hp_count = 0; | |||||
429 | uint64_t virt_addr; | |||||
430 | char buf[BUFSIZ8192]; | |||||
431 | char hugedir_str[PATH_MAX4096]; | |||||
432 | FILE *f; | |||||
433 | ||||||
434 | f = fopen("/proc/self/numa_maps", "r"); | |||||
435 | if (f == NULL((void*)0)) { | |||||
436 | RTE_LOG(NOTICE, EAL, "NUMA support not available"rte_log(6U, 0, "EAL" ": " "NUMA support not available" " consider that all memory is in socket_id 0\n" ) | |||||
437 | " consider that all memory is in socket_id 0\n")rte_log(6U, 0, "EAL" ": " "NUMA support not available" " consider that all memory is in socket_id 0\n" ); | |||||
438 | return 0; | |||||
439 | } | |||||
440 | ||||||
441 | snprintf(hugedir_str, sizeof(hugedir_str), | |||||
442 | "%s/%s", hpi->hugedir, eal_get_hugefile_prefix()); | |||||
443 | ||||||
444 | /* parse numa map */ | |||||
445 | while (fgets(buf, sizeof(buf), f) != NULL((void*)0)) { | |||||
446 | ||||||
447 | /* ignore non huge page */ | |||||
448 | if (strstr(buf, " huge ") == NULL((void*)0) && | |||||
449 | strstr(buf, hugedir_str) == NULL((void*)0)) | |||||
450 | continue; | |||||
451 | ||||||
452 | /* get zone addr */ | |||||
453 | virt_addr = strtoull(buf, &end, 16); | |||||
454 | if (virt_addr == 0 || end == buf) { | |||||
455 | RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n" , __func__); | |||||
456 | goto error; | |||||
457 | } | |||||
458 | ||||||
459 | /* get node id (socket id) */ | |||||
460 | nodestr = strstr(buf, " N"); | |||||
461 | if (nodestr == NULL((void*)0)) { | |||||
462 | RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n" , __func__); | |||||
463 | goto error; | |||||
464 | } | |||||
465 | nodestr += 2; | |||||
466 | end = strstr(nodestr, "="); | |||||
467 | if (end == NULL((void*)0)) { | |||||
468 | RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n" , __func__); | |||||
469 | goto error; | |||||
470 | } | |||||
471 | end[0] = '\0'; | |||||
472 | end = NULL((void*)0); | |||||
473 | ||||||
474 | socket_id = strtoul(nodestr, &end, 0); | |||||
475 | if ((nodestr[0] == '\0') || (end == NULL((void*)0)) || (*end != '\0')) { | |||||
476 | RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n" , __func__); | |||||
477 | goto error; | |||||
478 | } | |||||
479 | ||||||
480 | /* if we find this page in our mappings, set socket_id */ | |||||
481 | for (i = 0; i < hpi->num_pages[0]; i++) { | |||||
482 | void *va = (void *)(unsigned long)virt_addr; | |||||
483 | if (hugepg_tbl[i].orig_va == va) { | |||||
484 | hugepg_tbl[i].socket_id = socket_id; | |||||
485 | hp_count++; | |||||
486 | #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
487 | RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Hugepage %s is on socket %d\n", hugepg_tbl [i].filepath, socket_id) | |||||
488 | "Hugepage %s is on socket %d\n",rte_log(8U, 0, "EAL" ": " "Hugepage %s is on socket %d\n", hugepg_tbl [i].filepath, socket_id) | |||||
489 | hugepg_tbl[i].filepath, socket_id)rte_log(8U, 0, "EAL" ": " "Hugepage %s is on socket %d\n", hugepg_tbl [i].filepath, socket_id); | |||||
490 | #endif | |||||
491 | } | |||||
492 | } | |||||
493 | } | |||||
494 | ||||||
495 | if (hp_count < hpi->num_pages[0]) | |||||
496 | goto error; | |||||
497 | ||||||
498 | fclose(f); | |||||
499 | return 0; | |||||
500 | ||||||
501 | error: | |||||
502 | fclose(f); | |||||
503 | return -1; | |||||
504 | } | |||||
505 | ||||||
506 | static int | |||||
507 | cmp_physaddr(const void *a, const void *b) | |||||
508 | { | |||||
509 | #ifndef RTE_ARCH_PPC_64 | |||||
510 | const struct hugepage_file *p1 = a; | |||||
511 | const struct hugepage_file *p2 = b; | |||||
512 | #else | |||||
513 | /* PowerPC needs memory sorted in reverse order from x86 */ | |||||
514 | const struct hugepage_file *p1 = b; | |||||
515 | const struct hugepage_file *p2 = a; | |||||
516 | #endif | |||||
517 | if (p1->physaddr < p2->physaddr) | |||||
518 | return -1; | |||||
519 | else if (p1->physaddr > p2->physaddr) | |||||
520 | return 1; | |||||
521 | else | |||||
522 | return 0; | |||||
523 | } | |||||
524 | ||||||
525 | /* | |||||
526 | * Uses mmap to create a shared memory area for storage of data | |||||
527 | * Used in this file to store the hugepage file map on disk | |||||
528 | */ | |||||
529 | static void * | |||||
530 | create_shared_memory(const char *filename, const size_t mem_size) | |||||
531 | { | |||||
532 | void *retval; | |||||
533 | int fd; | |||||
534 | ||||||
535 | /* if no shared files mode is used, create anonymous memory instead */ | |||||
536 | if (internal_config.no_shconf) { | |||||
537 | retval = mmap(NULL((void*)0), mem_size, PROT_READ0x1 | PROT_WRITE0x2, | |||||
538 | MAP_PRIVATE0x02 | MAP_ANONYMOUS0x20, -1, 0); | |||||
539 | if (retval == MAP_FAILED((void *) -1)) | |||||
540 | return NULL((void*)0); | |||||
541 | return retval; | |||||
542 | } | |||||
543 | ||||||
544 | fd = open(filename, O_CREAT0100 | O_RDWR02, 0600); | |||||
545 | if (fd < 0) | |||||
546 | return NULL((void*)0); | |||||
547 | if (ftruncate(fd, mem_size) < 0) { | |||||
548 | close(fd); | |||||
549 | return NULL((void*)0); | |||||
550 | } | |||||
551 | retval = mmap(NULL((void*)0), mem_size, PROT_READ0x1 | PROT_WRITE0x2, MAP_SHARED0x01, fd, 0); | |||||
552 | close(fd); | |||||
553 | if (retval == MAP_FAILED((void *) -1)) | |||||
554 | return NULL((void*)0); | |||||
555 | return retval; | |||||
556 | } | |||||
557 | ||||||
558 | /* | |||||
559 | * this copies *active* hugepages from one hugepage table to another. | |||||
560 | * destination is typically the shared memory. | |||||
561 | */ | |||||
562 | static int | |||||
563 | copy_hugepages_to_shared_mem(struct hugepage_file * dst, int dest_size, | |||||
564 | const struct hugepage_file * src, int src_size) | |||||
565 | { | |||||
566 | int src_pos, dst_pos = 0; | |||||
567 | ||||||
568 | for (src_pos = 0; src_pos < src_size; src_pos++) { | |||||
569 | if (src[src_pos].orig_va != NULL((void*)0)) { | |||||
570 | /* error on overflow attempt */ | |||||
571 | if (dst_pos == dest_size) | |||||
572 | return -1; | |||||
573 | memcpy(&dst[dst_pos], &src[src_pos], sizeof(struct hugepage_file)); | |||||
574 | dst_pos++; | |||||
575 | } | |||||
576 | } | |||||
577 | return 0; | |||||
578 | } | |||||
579 | ||||||
580 | static int | |||||
581 | unlink_hugepage_files(struct hugepage_file *hugepg_tbl, | |||||
582 | unsigned num_hp_info) | |||||
583 | { | |||||
584 | unsigned socket, size; | |||||
585 | int page, nrpages = 0; | |||||
586 | ||||||
587 | /* get total number of hugepages */ | |||||
588 | for (size = 0; size < num_hp_info; size++) | |||||
589 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8; socket++) | |||||
590 | nrpages += | |||||
591 | internal_config.hugepage_info[size].num_pages[socket]; | |||||
592 | ||||||
593 | for (page = 0; page < nrpages; page++) { | |||||
594 | struct hugepage_file *hp = &hugepg_tbl[page]; | |||||
595 | ||||||
596 | if (hp->orig_va != NULL((void*)0) && unlink(hp->filepath)) { | |||||
597 | RTE_LOG(WARNING, EAL, "%s(): Removing %s failed: %s\n",rte_log(5U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__ , hp->filepath, strerror((*__errno_location ()))) | |||||
598 | __func__, hp->filepath, strerror(errno))rte_log(5U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__ , hp->filepath, strerror((*__errno_location ()))); | |||||
599 | } | |||||
600 | } | |||||
601 | return 0; | |||||
602 | } | |||||
603 | ||||||
604 | /* | |||||
605 | * unmaps hugepages that are not going to be used. since we originally allocate | |||||
606 | * ALL hugepages (not just those we need), additional unmapping needs to be done. | |||||
607 | */ | |||||
608 | static int | |||||
609 | unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl, | |||||
610 | struct hugepage_info *hpi, | |||||
611 | unsigned num_hp_info) | |||||
612 | { | |||||
613 | unsigned socket, size; | |||||
614 | int page, nrpages = 0; | |||||
615 | ||||||
616 | /* get total number of hugepages */ | |||||
617 | for (size = 0; size < num_hp_info; size++) | |||||
618 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8; socket++) | |||||
619 | nrpages += internal_config.hugepage_info[size].num_pages[socket]; | |||||
620 | ||||||
621 | for (size = 0; size < num_hp_info; size++) { | |||||
622 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8; socket++) { | |||||
623 | unsigned pages_found = 0; | |||||
624 | ||||||
625 | /* traverse until we have unmapped all the unused pages */ | |||||
626 | for (page = 0; page < nrpages; page++) { | |||||
627 | struct hugepage_file *hp = &hugepg_tbl[page]; | |||||
628 | ||||||
629 | /* find a page that matches the criteria */ | |||||
630 | if ((hp->size == hpi[size].hugepage_sz) && | |||||
631 | (hp->socket_id == (int) socket)) { | |||||
632 | ||||||
633 | /* if we skipped enough pages, unmap the rest */ | |||||
634 | if (pages_found == hpi[size].num_pages[socket]) { | |||||
635 | uint64_t unmap_len; | |||||
636 | ||||||
637 | unmap_len = hp->size; | |||||
638 | ||||||
639 | /* get start addr and len of the remaining segment */ | |||||
640 | munmap(hp->orig_va, | |||||
641 | (size_t)unmap_len); | |||||
642 | ||||||
643 | hp->orig_va = NULL((void*)0); | |||||
644 | if (unlink(hp->filepath) == -1) { | |||||
645 | RTE_LOG(ERR, EAL, "%s(): Removing %s failed: %s\n",rte_log(4U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__ , hp->filepath, strerror((*__errno_location ()))) | |||||
646 | __func__, hp->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__ , hp->filepath, strerror((*__errno_location ()))); | |||||
647 | return -1; | |||||
648 | } | |||||
649 | } else { | |||||
650 | /* lock the page and skip */ | |||||
651 | pages_found++; | |||||
652 | } | |||||
653 | ||||||
654 | } /* match page */ | |||||
655 | } /* foreach page */ | |||||
656 | } /* foreach socket */ | |||||
657 | } /* foreach pagesize */ | |||||
658 | ||||||
659 | return 0; | |||||
660 | } | |||||
661 | ||||||
662 | static int | |||||
663 | remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end) | |||||
664 | { | |||||
665 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |||||
666 | struct rte_memseg_list *msl; | |||||
667 | struct rte_fbarray *arr; | |||||
668 | int cur_page, seg_len; | |||||
669 | unsigned int msl_idx; | |||||
670 | int ms_idx; | |||||
671 | uint64_t page_sz; | |||||
672 | size_t memseg_len; | |||||
673 | int socket_id; | |||||
674 | ||||||
675 | page_sz = hugepages[seg_start].size; | |||||
676 | socket_id = hugepages[seg_start].socket_id; | |||||
677 | seg_len = seg_end - seg_start; | |||||
678 | ||||||
679 | RTE_LOG(DEBUG, EAL, "Attempting to map %" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Attempting to map %" "l" "u" "M on socket %i\n" , (seg_len * page_sz) >> 20ULL, socket_id) | |||||
680 | (seg_len * page_sz) >> 20ULL, socket_id)rte_log(8U, 0, "EAL" ": " "Attempting to map %" "l" "u" "M on socket %i\n" , (seg_len * page_sz) >> 20ULL, socket_id); | |||||
681 | ||||||
682 | /* find free space in memseg lists */ | |||||
683 | for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS64; msl_idx++) { | |||||
684 | bool_Bool empty; | |||||
685 | msl = &mcfg->memsegs[msl_idx]; | |||||
686 | arr = &msl->memseg_arr; | |||||
687 | ||||||
688 | if (msl->page_sz != page_sz) | |||||
689 | continue; | |||||
690 | if (msl->socket_id != socket_id) | |||||
691 | continue; | |||||
692 | ||||||
693 | /* leave space for a hole if array is not empty */ | |||||
694 | empty = arr->count == 0; | |||||
695 | ms_idx = rte_fbarray_find_next_n_free(arr, 0, | |||||
696 | seg_len + (empty ? 0 : 1)); | |||||
697 | ||||||
698 | /* memseg list is full? */ | |||||
699 | if (ms_idx < 0) | |||||
700 | continue; | |||||
701 | ||||||
702 | /* leave some space between memsegs, they are not IOVA | |||||
703 | * contiguous, so they shouldn't be VA contiguous either. | |||||
704 | */ | |||||
705 | if (!empty) | |||||
706 | ms_idx++; | |||||
707 | break; | |||||
708 | } | |||||
709 | if (msl_idx == RTE_MAX_MEMSEG_LISTS64) { | |||||
710 | RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase %s and/or %s in configuration.\n",rte_log(4U, 0, "EAL" ": " "Could not find space for memseg. Please increase %s and/or %s in configuration.\n" , "CONFIG_RTE_MAX_MEMSEG_PER_TYPE", "CONFIG_RTE_MAX_MEM_PER_TYPE" ) | |||||
711 | RTE_STR(CONFIG_RTE_MAX_MEMSEG_PER_TYPE),rte_log(4U, 0, "EAL" ": " "Could not find space for memseg. Please increase %s and/or %s in configuration.\n" , "CONFIG_RTE_MAX_MEMSEG_PER_TYPE", "CONFIG_RTE_MAX_MEM_PER_TYPE" ) | |||||
712 | RTE_STR(CONFIG_RTE_MAX_MEM_PER_TYPE))rte_log(4U, 0, "EAL" ": " "Could not find space for memseg. Please increase %s and/or %s in configuration.\n" , "CONFIG_RTE_MAX_MEMSEG_PER_TYPE", "CONFIG_RTE_MAX_MEM_PER_TYPE" ); | |||||
713 | return -1; | |||||
714 | } | |||||
715 | ||||||
716 | #ifdef RTE_ARCH_PPC64 | |||||
717 | /* for PPC64 we go through the list backwards */ | |||||
718 | for (cur_page = seg_end - 1; cur_page >= seg_start; | |||||
719 | cur_page--, ms_idx++) { | |||||
720 | #else | |||||
721 | for (cur_page = seg_start; cur_page < seg_end; cur_page++, ms_idx++) { | |||||
722 | #endif | |||||
723 | struct hugepage_file *hfile = &hugepages[cur_page]; | |||||
724 | struct rte_memseg *ms = rte_fbarray_get(arr, ms_idx); | |||||
725 | void *addr; | |||||
726 | int fd; | |||||
727 | ||||||
728 | fd = open(hfile->filepath, O_RDWR02); | |||||
729 | if (fd < 0) { | |||||
730 | RTE_LOG(ERR, EAL, "Could not open '%s': %s\n",rte_log(4U, 0, "EAL" ": " "Could not open '%s': %s\n", hfile-> filepath, strerror((*__errno_location ()))) | |||||
731 | hfile->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Could not open '%s': %s\n", hfile-> filepath, strerror((*__errno_location ()))); | |||||
732 | return -1; | |||||
733 | } | |||||
734 | /* set shared lock on the file. */ | |||||
735 | if (flock(fd, LOCK_SH1) < 0) { | |||||
736 | RTE_LOG(DEBUG, EAL, "Could not lock '%s': %s\n",rte_log(8U, 0, "EAL" ": " "Could not lock '%s': %s\n", hfile-> filepath, strerror((*__errno_location ()))) | |||||
737 | hfile->filepath, strerror(errno))rte_log(8U, 0, "EAL" ": " "Could not lock '%s': %s\n", hfile-> filepath, strerror((*__errno_location ()))); | |||||
738 | close(fd); | |||||
739 | return -1; | |||||
740 | } | |||||
741 | memseg_len = (size_t)page_sz; | |||||
742 | addr = RTE_PTR_ADD(msl->base_va, ms_idx * memseg_len)((void*)((uintptr_t)(msl->base_va) + (ms_idx * memseg_len) )); | |||||
743 | ||||||
744 | /* we know this address is already mmapped by memseg list, so | |||||
745 | * using MAP_FIXED here is safe | |||||
746 | */ | |||||
747 | addr = mmap(addr, page_sz, PROT_READ0x1 | PROT_WRITE0x2, | |||||
748 | MAP_SHARED0x01 | MAP_POPULATE0x08000 | MAP_FIXED0x10, fd, 0); | |||||
749 | if (addr == MAP_FAILED((void *) -1)) { | |||||
750 | RTE_LOG(ERR, EAL, "Couldn't remap '%s': %s\n",rte_log(4U, 0, "EAL" ": " "Couldn't remap '%s': %s\n", hfile-> filepath, strerror((*__errno_location ()))) | |||||
751 | hfile->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Couldn't remap '%s': %s\n", hfile-> filepath, strerror((*__errno_location ()))); | |||||
752 | close(fd); | |||||
753 | return -1; | |||||
754 | } | |||||
755 | ||||||
756 | /* we have a new address, so unmap previous one */ | |||||
757 | #ifndef RTE_ARCH_641 | |||||
758 | /* in 32-bit legacy mode, we have already unmapped the page */ | |||||
759 | if (!internal_config.legacy_mem) | |||||
760 | munmap(hfile->orig_va, page_sz); | |||||
761 | #else | |||||
762 | munmap(hfile->orig_va, page_sz); | |||||
763 | #endif | |||||
764 | ||||||
765 | hfile->orig_va = NULL((void*)0); | |||||
766 | hfile->final_va = addr; | |||||
767 | ||||||
768 | /* rewrite physical addresses in IOVA as VA mode */ | |||||
769 | if (rte_eal_iova_mode() == RTE_IOVA_VA) | |||||
770 | hfile->physaddr = (uintptr_t)addr; | |||||
771 | ||||||
772 | /* set up memseg data */ | |||||
773 | ms->addr = addr; | |||||
774 | ms->hugepage_sz = page_sz; | |||||
775 | ms->len = memseg_len; | |||||
776 | ms->iova = hfile->physaddr; | |||||
777 | ms->socket_id = hfile->socket_id; | |||||
778 | ms->nchannel = rte_memory_get_nchannel(); | |||||
779 | ms->nrank = rte_memory_get_nrank(); | |||||
780 | ||||||
781 | rte_fbarray_set_used(arr, ms_idx); | |||||
782 | ||||||
783 | /* store segment fd internally */ | |||||
784 | if (eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd) < 0) | |||||
785 | RTE_LOG(ERR, EAL, "Could not store segment fd: %s\n",rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n", rte_strerror((per_lcore__rte_errno))) | |||||
786 | rte_strerror(rte_errno))rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n", rte_strerror((per_lcore__rte_errno))); | |||||
787 | } | |||||
788 | RTE_LOG(DEBUG, EAL, "Allocated %" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Allocated %" "l" "u" "M on socket %i\n" , (seg_len * page_sz) >> 20, socket_id) | |||||
789 | (seg_len * page_sz) >> 20, socket_id)rte_log(8U, 0, "EAL" ": " "Allocated %" "l" "u" "M on socket %i\n" , (seg_len * page_sz) >> 20, socket_id); | |||||
790 | return 0; | |||||
791 | } | |||||
792 | ||||||
793 | static uint64_t | |||||
794 | get_mem_amount(uint64_t page_sz, uint64_t max_mem) | |||||
795 | { | |||||
796 | uint64_t area_sz, max_pages; | |||||
797 | ||||||
798 | /* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */ | |||||
799 | max_pages = RTE_MAX_MEMSEG_PER_LIST8192; | |||||
800 | max_mem = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20, max_mem)__extension__ ({ __typeof__ ((uint64_t)32768 << 20) _a = ((uint64_t)32768 << 20); __typeof__ (max_mem) _b = (max_mem ); _a < _b ? _a : _b; }); | |||||
801 | ||||||
802 | area_sz = RTE_MIN(page_sz * max_pages, max_mem)__extension__ ({ __typeof__ (page_sz * max_pages) _a = (page_sz * max_pages); __typeof__ (max_mem) _b = (max_mem); _a < _b ? _a : _b; }); | |||||
803 | ||||||
804 | /* make sure the list isn't smaller than the page size */ | |||||
805 | area_sz = RTE_MAX(area_sz, page_sz)__extension__ ({ __typeof__ (area_sz) _a = (area_sz); __typeof__ (page_sz) _b = (page_sz); _a > _b ? _a : _b; }); | |||||
806 | ||||||
807 | return RTE_ALIGN(area_sz, page_sz)(__typeof__(((area_sz) + ((__typeof__(area_sz)) (page_sz) - 1 ))))((((area_sz) + ((__typeof__(area_sz)) (page_sz) - 1))) & (~((__typeof__(((area_sz) + ((__typeof__(area_sz)) (page_sz) - 1))))((page_sz) - 1)))); | |||||
808 | } | |||||
809 | ||||||
810 | static int | |||||
811 | free_memseg_list(struct rte_memseg_list *msl) | |||||
812 | { | |||||
813 | if (rte_fbarray_destroy(&msl->memseg_arr)) { | |||||
814 | RTE_LOG(ERR, EAL, "Cannot destroy memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot destroy memseg list\n"); | |||||
815 | return -1; | |||||
816 | } | |||||
817 | memset(msl, 0, sizeof(*msl)); | |||||
818 | return 0; | |||||
819 | } | |||||
820 | ||||||
821 | #define MEMSEG_LIST_FMT"memseg-%" "l" "u" "k-%i-%i" "memseg-%" PRIu64"l" "u" "k-%i-%i" | |||||
822 | static int | |||||
823 | alloc_memseg_list(struct rte_memseg_list *msl, uint64_t page_sz, | |||||
824 | int n_segs, int socket_id, int type_msl_idx) | |||||
825 | { | |||||
826 | char name[RTE_FBARRAY_NAME_LEN64]; | |||||
827 | ||||||
828 | snprintf(name, sizeof(name), MEMSEG_LIST_FMT"memseg-%" "l" "u" "k-%i-%i", page_sz >> 10, socket_id, | |||||
829 | type_msl_idx); | |||||
830 | if (rte_fbarray_init(&msl->memseg_arr, name, n_segs, | |||||
831 | sizeof(struct rte_memseg))) { | |||||
832 | RTE_LOG(ERR, EAL, "Cannot allocate memseg list: %s\n",rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list: %s\n" , rte_strerror((per_lcore__rte_errno))) | |||||
833 | rte_strerror(rte_errno))rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list: %s\n" , rte_strerror((per_lcore__rte_errno))); | |||||
834 | return -1; | |||||
835 | } | |||||
836 | ||||||
837 | msl->page_sz = page_sz; | |||||
838 | msl->socket_id = socket_id; | |||||
839 | msl->base_va = NULL((void*)0); | |||||
840 | ||||||
841 | RTE_LOG(DEBUG, EAL, "Memseg list allocated: 0x%zxkB at socket %i\n",rte_log(8U, 0, "EAL" ": " "Memseg list allocated: 0x%zxkB at socket %i\n" , (size_t)page_sz >> 10, socket_id) | |||||
842 | (size_t)page_sz >> 10, socket_id)rte_log(8U, 0, "EAL" ": " "Memseg list allocated: 0x%zxkB at socket %i\n" , (size_t)page_sz >> 10, socket_id); | |||||
843 | ||||||
844 | return 0; | |||||
845 | } | |||||
846 | ||||||
847 | static int | |||||
848 | alloc_va_space(struct rte_memseg_list *msl) | |||||
849 | { | |||||
850 | uint64_t page_sz; | |||||
851 | size_t mem_sz; | |||||
852 | void *addr; | |||||
853 | int flags = 0; | |||||
854 | ||||||
855 | page_sz = msl->page_sz; | |||||
856 | mem_sz = page_sz * msl->memseg_arr.len; | |||||
857 | ||||||
858 | addr = eal_get_virtual_area(msl->base_va, &mem_sz, page_sz, 0, flags); | |||||
859 | if (addr == NULL((void*)0)) { | |||||
860 | if (rte_errno(per_lcore__rte_errno) == EADDRNOTAVAIL99) | |||||
861 | RTE_LOG(ERR, EAL, "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n",rte_log(4U, 0, "EAL" ": " "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n" , (unsigned long long)mem_sz, msl->base_va) | |||||
862 | (unsigned long long)mem_sz, msl->base_va)rte_log(4U, 0, "EAL" ": " "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n" , (unsigned long long)mem_sz, msl->base_va); | |||||
863 | else | |||||
864 | RTE_LOG(ERR, EAL, "Cannot reserve memory\n")rte_log(4U, 0, "EAL" ": " "Cannot reserve memory\n"); | |||||
865 | return -1; | |||||
866 | } | |||||
867 | msl->base_va = addr; | |||||
868 | msl->len = mem_sz; | |||||
869 | ||||||
870 | return 0; | |||||
871 | } | |||||
872 | ||||||
873 | /* | |||||
874 | * Our VA space is not preallocated yet, so preallocate it here. We need to know | |||||
875 | * how many segments there are in order to map all pages into one address space, | |||||
876 | * and leave appropriate holes between segments so that rte_malloc does not | |||||
877 | * concatenate them into one big segment. | |||||
878 | * | |||||
879 | * we also need to unmap original pages to free up address space. | |||||
880 | */ | |||||
881 | static int __rte_unused__attribute__((__unused__)) | |||||
882 | prealloc_segments(struct hugepage_file *hugepages, int n_pages) | |||||
883 | { | |||||
884 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |||||
885 | int cur_page, seg_start_page, end_seg, new_memseg; | |||||
886 | unsigned int hpi_idx, socket, i; | |||||
887 | int n_contig_segs, n_segs; | |||||
888 | int msl_idx; | |||||
889 | ||||||
890 | /* before we preallocate segments, we need to free up our VA space. | |||||
891 | * we're not removing files, and we already have information about | |||||
892 | * PA-contiguousness, so it is safe to unmap everything. | |||||
893 | */ | |||||
894 | for (cur_page = 0; cur_page < n_pages; cur_page++) { | |||||
895 | struct hugepage_file *hpi = &hugepages[cur_page]; | |||||
896 | munmap(hpi->orig_va, hpi->size); | |||||
897 | hpi->orig_va = NULL((void*)0); | |||||
898 | } | |||||
899 | ||||||
900 | /* we cannot know how many page sizes and sockets we have discovered, so | |||||
901 | * loop over all of them | |||||
902 | */ | |||||
903 | for (hpi_idx = 0; hpi_idx < internal_config.num_hugepage_sizes; | |||||
904 | hpi_idx++) { | |||||
905 | uint64_t page_sz = | |||||
906 | internal_config.hugepage_info[hpi_idx].hugepage_sz; | |||||
907 | ||||||
908 | for (i = 0; i < rte_socket_count(); i++) { | |||||
909 | struct rte_memseg_list *msl; | |||||
910 | ||||||
911 | socket = rte_socket_id_by_idx(i); | |||||
912 | n_contig_segs = 0; | |||||
913 | n_segs = 0; | |||||
914 | seg_start_page = -1; | |||||
915 | ||||||
916 | for (cur_page = 0; cur_page < n_pages; cur_page++) { | |||||
917 | struct hugepage_file *prev, *cur; | |||||
918 | int prev_seg_start_page = -1; | |||||
919 | ||||||
920 | cur = &hugepages[cur_page]; | |||||
921 | prev = cur_page == 0 ? NULL((void*)0) : | |||||
922 | &hugepages[cur_page - 1]; | |||||
923 | ||||||
924 | new_memseg = 0; | |||||
925 | end_seg = 0; | |||||
926 | ||||||
927 | if (cur->size == 0) | |||||
928 | end_seg = 1; | |||||
929 | else if (cur->socket_id != (int) socket) | |||||
930 | end_seg = 1; | |||||
931 | else if (cur->size != page_sz) | |||||
932 | end_seg = 1; | |||||
933 | else if (cur_page == 0) | |||||
934 | new_memseg = 1; | |||||
935 | #ifdef RTE_ARCH_PPC_64 | |||||
936 | /* On PPC64 architecture, the mmap always start | |||||
937 | * from higher address to lower address. Here, | |||||
938 | * physical addresses are in descending order. | |||||
939 | */ | |||||
940 | else if ((prev->physaddr - cur->physaddr) != | |||||
941 | cur->size) | |||||
942 | new_memseg = 1; | |||||
943 | #else | |||||
944 | else if ((cur->physaddr - prev->physaddr) != | |||||
945 | cur->size) | |||||
946 | new_memseg = 1; | |||||
947 | #endif | |||||
948 | if (new_memseg) { | |||||
949 | /* if we're already inside a segment, | |||||
950 | * new segment means end of current one | |||||
951 | */ | |||||
952 | if (seg_start_page != -1) { | |||||
953 | end_seg = 1; | |||||
954 | prev_seg_start_page = | |||||
955 | seg_start_page; | |||||
956 | } | |||||
957 | seg_start_page = cur_page; | |||||
958 | } | |||||
959 | ||||||
960 | if (end_seg) { | |||||
961 | if (prev_seg_start_page != -1) { | |||||
962 | /* we've found a new segment */ | |||||
963 | n_contig_segs++; | |||||
964 | n_segs += cur_page - | |||||
965 | prev_seg_start_page; | |||||
966 | } else if (seg_start_page != -1) { | |||||
967 | /* we didn't find new segment, | |||||
968 | * but did end current one | |||||
969 | */ | |||||
970 | n_contig_segs++; | |||||
971 | n_segs += cur_page - | |||||
972 | seg_start_page; | |||||
973 | seg_start_page = -1; | |||||
974 | continue; | |||||
975 | } else { | |||||
976 | /* we're skipping this page */ | |||||
977 | continue; | |||||
978 | } | |||||
979 | } | |||||
980 | /* segment continues */ | |||||
981 | } | |||||
982 | /* check if we missed last segment */ | |||||
983 | if (seg_start_page != -1) { | |||||
984 | n_contig_segs++; | |||||
985 | n_segs += cur_page - seg_start_page; | |||||
986 | } | |||||
987 | ||||||
988 | /* if no segments were found, do not preallocate */ | |||||
989 | if (n_segs == 0) | |||||
990 | continue; | |||||
991 | ||||||
992 | /* we now have total number of pages that we will | |||||
993 | * allocate for this segment list. add separator pages | |||||
994 | * to the total count, and preallocate VA space. | |||||
995 | */ | |||||
996 | n_segs += n_contig_segs - 1; | |||||
997 | ||||||
998 | /* now, preallocate VA space for these segments */ | |||||
999 | ||||||
1000 | /* first, find suitable memseg list for this */ | |||||
1001 | for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS64; | |||||
1002 | msl_idx++) { | |||||
1003 | msl = &mcfg->memsegs[msl_idx]; | |||||
1004 | ||||||
1005 | if (msl->base_va != NULL((void*)0)) | |||||
1006 | continue; | |||||
1007 | break; | |||||
1008 | } | |||||
1009 | if (msl_idx == RTE_MAX_MEMSEG_LISTS64) { | |||||
1010 | RTE_LOG(ERR, EAL, "Not enough space in memseg lists, please increase %s\n",rte_log(4U, 0, "EAL" ": " "Not enough space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS") | |||||
1011 | RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "Not enough space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS"); | |||||
1012 | return -1; | |||||
1013 | } | |||||
1014 | ||||||
1015 | /* now, allocate fbarray itself */ | |||||
1016 | if (alloc_memseg_list(msl, page_sz, n_segs, socket, | |||||
1017 | msl_idx) < 0) | |||||
1018 | return -1; | |||||
1019 | ||||||
1020 | /* finally, allocate VA space */ | |||||
1021 | if (alloc_va_space(msl) < 0) | |||||
1022 | return -1; | |||||
1023 | } | |||||
1024 | } | |||||
1025 | return 0; | |||||
1026 | } | |||||
1027 | ||||||
1028 | /* | |||||
1029 | * We cannot reallocate memseg lists on the fly because PPC64 stores pages | |||||
1030 | * backwards, therefore we have to process the entire memseg first before | |||||
1031 | * remapping it into memseg list VA space. | |||||
1032 | */ | |||||
1033 | static int | |||||
1034 | remap_needed_hugepages(struct hugepage_file *hugepages, int n_pages) | |||||
1035 | { | |||||
1036 | int cur_page, seg_start_page, new_memseg, ret; | |||||
1037 | ||||||
1038 | seg_start_page = 0; | |||||
1039 | for (cur_page = 0; cur_page < n_pages; cur_page++) { | |||||
1040 | struct hugepage_file *prev, *cur; | |||||
1041 | ||||||
1042 | new_memseg = 0; | |||||
1043 | ||||||
1044 | cur = &hugepages[cur_page]; | |||||
1045 | prev = cur_page == 0 ? NULL((void*)0) : &hugepages[cur_page - 1]; | |||||
1046 | ||||||
1047 | /* if size is zero, no more pages left */ | |||||
1048 | if (cur->size == 0) | |||||
1049 | break; | |||||
1050 | ||||||
1051 | if (cur_page == 0) | |||||
1052 | new_memseg = 1; | |||||
1053 | else if (cur->socket_id != prev->socket_id) | |||||
1054 | new_memseg = 1; | |||||
1055 | else if (cur->size != prev->size) | |||||
1056 | new_memseg = 1; | |||||
1057 | #ifdef RTE_ARCH_PPC_64 | |||||
1058 | /* On PPC64 architecture, the mmap always start from higher | |||||
1059 | * address to lower address. Here, physical addresses are in | |||||
1060 | * descending order. | |||||
1061 | */ | |||||
1062 | else if ((prev->physaddr - cur->physaddr) != cur->size) | |||||
1063 | new_memseg = 1; | |||||
1064 | #else | |||||
1065 | else if ((cur->physaddr - prev->physaddr) != cur->size) | |||||
1066 | new_memseg = 1; | |||||
1067 | #endif | |||||
1068 | ||||||
1069 | if (new_memseg) { | |||||
1070 | /* if this isn't the first time, remap segment */ | |||||
1071 | if (cur_page != 0) { | |||||
1072 | ret = remap_segment(hugepages, seg_start_page, | |||||
1073 | cur_page); | |||||
1074 | if (ret != 0) | |||||
1075 | return -1; | |||||
1076 | } | |||||
1077 | /* remember where we started */ | |||||
1078 | seg_start_page = cur_page; | |||||
1079 | } | |||||
1080 | /* continuation of previous memseg */ | |||||
1081 | } | |||||
1082 | /* we were stopped, but we didn't remap the last segment, do it now */ | |||||
1083 | if (cur_page != 0) { | |||||
1084 | ret = remap_segment(hugepages, seg_start_page, | |||||
1085 | cur_page); | |||||
1086 | if (ret != 0) | |||||
1087 | return -1; | |||||
1088 | } | |||||
1089 | return 0; | |||||
1090 | } | |||||
1091 | ||||||
1092 | static inline uint64_t | |||||
1093 | get_socket_mem_size(int socket) | |||||
1094 | { | |||||
1095 | uint64_t size = 0; | |||||
1096 | unsigned i; | |||||
1097 | ||||||
1098 | for (i = 0; i < internal_config.num_hugepage_sizes; i++){ | |||||
1099 | struct hugepage_info *hpi = &internal_config.hugepage_info[i]; | |||||
1100 | size += hpi->hugepage_sz * hpi->num_pages[socket]; | |||||
1101 | } | |||||
1102 | ||||||
1103 | return size; | |||||
1104 | } | |||||
1105 | ||||||
1106 | /* | |||||
1107 | * This function is a NUMA-aware equivalent of calc_num_pages. | |||||
1108 | * It takes in the list of hugepage sizes and the | |||||
1109 | * number of pages thereof, and calculates the best number of | |||||
1110 | * pages of each size to fulfill the request for <memory> ram | |||||
1111 | */ | |||||
1112 | static int | |||||
1113 | calc_num_pages_per_socket(uint64_t * memory, | |||||
1114 | struct hugepage_info *hp_info, | |||||
1115 | struct hugepage_info *hp_used, | |||||
1116 | unsigned num_hp_info) | |||||
1117 | { | |||||
1118 | unsigned socket, j, i = 0; | |||||
1119 | unsigned requested, available; | |||||
1120 | int total_num_pages = 0; | |||||
1121 | uint64_t remaining_mem, cur_mem; | |||||
1122 | uint64_t total_mem = internal_config.memory; | |||||
1123 | ||||||
1124 | if (num_hp_info == 0) | |||||
1125 | return -1; | |||||
1126 | ||||||
1127 | /* if specific memory amounts per socket weren't requested */ | |||||
1128 | if (internal_config.force_sockets == 0) { | |||||
1129 | size_t total_size; | |||||
1130 | #ifdef RTE_ARCH_641 | |||||
1131 | int cpu_per_socket[RTE_MAX_NUMA_NODES8]; | |||||
1132 | size_t default_size; | |||||
1133 | unsigned lcore_id; | |||||
1134 | ||||||
1135 | /* Compute number of cores per socket */ | |||||
1136 | memset(cpu_per_socket, 0, sizeof(cpu_per_socket)); | |||||
1137 | RTE_LCORE_FOREACH(lcore_id)for (lcore_id = rte_get_next_lcore(-1, 0, 0); lcore_id<128 ; lcore_id = rte_get_next_lcore(lcore_id, 0, 0)) { | |||||
1138 | cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++; | |||||
1139 | } | |||||
1140 | ||||||
1141 | /* | |||||
1142 | * Automatically spread requested memory amongst detected sockets according | |||||
1143 | * to number of cores from cpu mask present on each socket | |||||
1144 | */ | |||||
1145 | total_size = internal_config.memory; | |||||
1146 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_size != 0; socket++) { | |||||
1147 | ||||||
1148 | /* Set memory amount per socket */ | |||||
1149 | default_size = (internal_config.memory * cpu_per_socket[socket]) | |||||
1150 | / rte_lcore_count(); | |||||
1151 | ||||||
1152 | /* Limit to maximum available memory on socket */ | |||||
1153 | default_size = RTE_MIN(default_size, get_socket_mem_size(socket))__extension__ ({ __typeof__ (default_size) _a = (default_size ); __typeof__ (get_socket_mem_size(socket)) _b = (get_socket_mem_size (socket)); _a < _b ? _a : _b; }); | |||||
1154 | ||||||
1155 | /* Update sizes */ | |||||
1156 | memory[socket] = default_size; | |||||
1157 | total_size -= default_size; | |||||
1158 | } | |||||
1159 | ||||||
1160 | /* | |||||
1161 | * If some memory is remaining, try to allocate it by getting all | |||||
1162 | * available memory from sockets, one after the other | |||||
1163 | */ | |||||
1164 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_size != 0; socket++) { | |||||
1165 | /* take whatever is available */ | |||||
1166 | default_size = RTE_MIN(get_socket_mem_size(socket) - memory[socket],__extension__ ({ __typeof__ (get_socket_mem_size(socket) - memory [socket]) _a = (get_socket_mem_size(socket) - memory[socket]) ; __typeof__ (total_size) _b = (total_size); _a < _b ? _a : _b; }) | |||||
1167 | total_size)__extension__ ({ __typeof__ (get_socket_mem_size(socket) - memory [socket]) _a = (get_socket_mem_size(socket) - memory[socket]) ; __typeof__ (total_size) _b = (total_size); _a < _b ? _a : _b; }); | |||||
1168 | ||||||
1169 | /* Update sizes */ | |||||
1170 | memory[socket] += default_size; | |||||
1171 | total_size -= default_size; | |||||
1172 | } | |||||
1173 | #else | |||||
1174 | /* in 32-bit mode, allocate all of the memory only on master | |||||
1175 | * lcore socket | |||||
1176 | */ | |||||
1177 | total_size = internal_config.memory; | |||||
1178 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_size != 0; | |||||
1179 | socket++) { | |||||
1180 | struct rte_config *cfg = rte_eal_get_configuration(); | |||||
1181 | unsigned int master_lcore_socket; | |||||
1182 | ||||||
1183 | master_lcore_socket = | |||||
1184 | rte_lcore_to_socket_id(cfg->master_lcore); | |||||
1185 | ||||||
1186 | if (master_lcore_socket != socket) | |||||
1187 | continue; | |||||
1188 | ||||||
1189 | /* Update sizes */ | |||||
1190 | memory[socket] = total_size; | |||||
1191 | break; | |||||
1192 | } | |||||
1193 | #endif | |||||
1194 | } | |||||
1195 | ||||||
1196 | for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_mem != 0; socket++) { | |||||
1197 | /* skips if the memory on specific socket wasn't requested */ | |||||
1198 | for (i = 0; i < num_hp_info && memory[socket] != 0; i++){ | |||||
1199 | strlcpy(hp_used[i].hugedir, hp_info[i].hugedir,rte_strlcpy(hp_used[i].hugedir, hp_info[i].hugedir, sizeof(hp_used [i].hugedir)) | |||||
1200 | sizeof(hp_used[i].hugedir))rte_strlcpy(hp_used[i].hugedir, hp_info[i].hugedir, sizeof(hp_used [i].hugedir)); | |||||
1201 | hp_used[i].num_pages[socket] = RTE_MIN(__extension__ ({ __typeof__ (memory[socket] / hp_info[i].hugepage_sz ) _a = (memory[socket] / hp_info[i].hugepage_sz); __typeof__ ( hp_info[i].num_pages[socket]) _b = (hp_info[i].num_pages[socket ]); _a < _b ? _a : _b; }) | |||||
1202 | memory[socket] / hp_info[i].hugepage_sz,__extension__ ({ __typeof__ (memory[socket] / hp_info[i].hugepage_sz ) _a = (memory[socket] / hp_info[i].hugepage_sz); __typeof__ ( hp_info[i].num_pages[socket]) _b = (hp_info[i].num_pages[socket ]); _a < _b ? _a : _b; }) | |||||
1203 | hp_info[i].num_pages[socket])__extension__ ({ __typeof__ (memory[socket] / hp_info[i].hugepage_sz ) _a = (memory[socket] / hp_info[i].hugepage_sz); __typeof__ ( hp_info[i].num_pages[socket]) _b = (hp_info[i].num_pages[socket ]); _a < _b ? _a : _b; }); | |||||
1204 | ||||||
1205 | cur_mem = hp_used[i].num_pages[socket] * | |||||
1206 | hp_used[i].hugepage_sz; | |||||
1207 | ||||||
1208 | memory[socket] -= cur_mem; | |||||
1209 | total_mem -= cur_mem; | |||||
1210 | ||||||
1211 | total_num_pages += hp_used[i].num_pages[socket]; | |||||
1212 | ||||||
1213 | /* check if we have met all memory requests */ | |||||
1214 | if (memory[socket] == 0) | |||||
1215 | break; | |||||
1216 | ||||||
1217 | /* check if we have any more pages left at this size, if so | |||||
1218 | * move on to next size */ | |||||
1219 | if (hp_used[i].num_pages[socket] == hp_info[i].num_pages[socket]) | |||||
1220 | continue; | |||||
1221 | /* At this point we know that there are more pages available that are | |||||
1222 | * bigger than the memory we want, so lets see if we can get enough | |||||
1223 | * from other page sizes. | |||||
1224 | */ | |||||
1225 | remaining_mem = 0; | |||||
1226 | for (j = i+1; j < num_hp_info; j++) | |||||
1227 | remaining_mem += hp_info[j].hugepage_sz * | |||||
1228 | hp_info[j].num_pages[socket]; | |||||
1229 | ||||||
1230 | /* is there enough other memory, if not allocate another page and quit */ | |||||
1231 | if (remaining_mem < memory[socket]){ | |||||
1232 | cur_mem = RTE_MIN(memory[socket],__extension__ ({ __typeof__ (memory[socket]) _a = (memory[socket ]); __typeof__ (hp_info[i].hugepage_sz) _b = (hp_info[i].hugepage_sz ); _a < _b ? _a : _b; }) | |||||
1233 | hp_info[i].hugepage_sz)__extension__ ({ __typeof__ (memory[socket]) _a = (memory[socket ]); __typeof__ (hp_info[i].hugepage_sz) _b = (hp_info[i].hugepage_sz ); _a < _b ? _a : _b; }); | |||||
1234 | memory[socket] -= cur_mem; | |||||
1235 | total_mem -= cur_mem; | |||||
1236 | hp_used[i].num_pages[socket]++; | |||||
1237 | total_num_pages++; | |||||
1238 | break; /* we are done with this socket*/ | |||||
1239 | } | |||||
1240 | } | |||||
1241 | /* if we didn't satisfy all memory requirements per socket */ | |||||
1242 | if (memory[socket] > 0 && | |||||
1243 | internal_config.socket_mem[socket] != 0) { | |||||
1244 | /* to prevent icc errors */ | |||||
1245 | requested = (unsigned) (internal_config.socket_mem[socket] / | |||||
1246 | 0x100000); | |||||
1247 | available = requested - | |||||
1248 | ((unsigned) (memory[socket] / 0x100000)); | |||||
1249 | RTE_LOG(ERR, EAL, "Not enough memory available on socket %u! "rte_log(4U, 0, "EAL" ": " "Not enough memory available on socket %u! " "Requested: %uMB, available: %uMB\n", socket, requested, available ) | |||||
1250 | "Requested: %uMB, available: %uMB\n", socket,rte_log(4U, 0, "EAL" ": " "Not enough memory available on socket %u! " "Requested: %uMB, available: %uMB\n", socket, requested, available ) | |||||
1251 | requested, available)rte_log(4U, 0, "EAL" ": " "Not enough memory available on socket %u! " "Requested: %uMB, available: %uMB\n", socket, requested, available ); | |||||
1252 | return -1; | |||||
1253 | } | |||||
1254 | } | |||||
1255 | ||||||
1256 | /* if we didn't satisfy total memory requirements */ | |||||
1257 | if (total_mem > 0) { | |||||
1258 | requested = (unsigned) (internal_config.memory / 0x100000); | |||||
1259 | available = requested - (unsigned) (total_mem / 0x100000); | |||||
1260 | RTE_LOG(ERR, EAL, "Not enough memory available! Requested: %uMB,"rte_log(4U, 0, "EAL" ": " "Not enough memory available! Requested: %uMB," " available: %uMB\n", requested, available) | |||||
1261 | " available: %uMB\n", requested, available)rte_log(4U, 0, "EAL" ": " "Not enough memory available! Requested: %uMB," " available: %uMB\n", requested, available); | |||||
1262 | return -1; | |||||
1263 | } | |||||
1264 | return total_num_pages; | |||||
1265 | } | |||||
1266 | ||||||
1267 | static inline size_t | |||||
1268 | eal_get_hugepage_mem_size(void) | |||||
1269 | { | |||||
1270 | uint64_t size = 0; | |||||
1271 | unsigned i, j; | |||||
1272 | ||||||
1273 | for (i = 0; i < internal_config.num_hugepage_sizes; i++) { | |||||
1274 | struct hugepage_info *hpi = &internal_config.hugepage_info[i]; | |||||
1275 | if (strnlen(hpi->hugedir, sizeof(hpi->hugedir)) != 0) { | |||||
1276 | for (j = 0; j < RTE_MAX_NUMA_NODES8; j++) { | |||||
1277 | size += hpi->hugepage_sz * hpi->num_pages[j]; | |||||
1278 | } | |||||
1279 | } | |||||
1280 | } | |||||
1281 | ||||||
1282 | return (size < SIZE_MAX(18446744073709551615UL)) ? (size_t)(size) : SIZE_MAX(18446744073709551615UL); | |||||
1283 | } | |||||
1284 | ||||||
1285 | static struct sigaction huge_action_old; | |||||
1286 | static int huge_need_recover; | |||||
1287 | ||||||
1288 | static void | |||||
1289 | huge_register_sigbus(void) | |||||
1290 | { | |||||
1291 | sigset_t mask; | |||||
1292 | struct sigaction action; | |||||
1293 | ||||||
1294 | sigemptyset(&mask); | |||||
1295 | sigaddset(&mask, SIGBUS7); | |||||
1296 | action.sa_flags = 0; | |||||
1297 | action.sa_mask = mask; | |||||
1298 | action.sa_handler__sigaction_handler.sa_handler = huge_sigbus_handler; | |||||
1299 | ||||||
1300 | huge_need_recover = !sigaction(SIGBUS7, &action, &huge_action_old); | |||||
1301 | } | |||||
1302 | ||||||
1303 | static void | |||||
1304 | huge_recover_sigbus(void) | |||||
1305 | { | |||||
1306 | if (huge_need_recover) { | |||||
1307 | sigaction(SIGBUS7, &huge_action_old, NULL((void*)0)); | |||||
1308 | huge_need_recover = 0; | |||||
1309 | } | |||||
1310 | } | |||||
1311 | ||||||
1312 | /* | |||||
1313 | * Prepare physical memory mapping: fill configuration structure with | |||||
1314 | * these infos, return 0 on success. | |||||
1315 | * 1. map N huge pages in separate files in hugetlbfs | |||||
1316 | * 2. find associated physical addr | |||||
1317 | * 3. find associated NUMA socket ID | |||||
1318 | * 4. sort all huge pages by physical address | |||||
1319 | * 5. remap these N huge pages in the correct order | |||||
1320 | * 6. unmap the first mapping | |||||
1321 | * 7. fill memsegs in configuration with contiguous zones | |||||
1322 | */ | |||||
1323 | static int | |||||
1324 | eal_legacy_hugepage_init(void) | |||||
1325 | { | |||||
1326 | struct rte_mem_config *mcfg; | |||||
1327 | struct hugepage_file *hugepage = NULL((void*)0), *tmp_hp = NULL((void*)0); | |||||
1328 | struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES3]; | |||||
1329 | struct rte_fbarray *arr; | |||||
1330 | struct rte_memseg *ms; | |||||
1331 | ||||||
1332 | uint64_t memory[RTE_MAX_NUMA_NODES8]; | |||||
1333 | ||||||
1334 | unsigned hp_offset; | |||||
1335 | int i, j; | |||||
1336 | int nr_hugefiles, nr_hugepages = 0; | |||||
1337 | void *addr; | |||||
1338 | ||||||
1339 | test_phys_addrs_available(); | |||||
1340 | ||||||
1341 | memset(used_hp, 0, sizeof(used_hp)); | |||||
1342 | ||||||
1343 | /* get pointer to global configuration */ | |||||
1344 | mcfg = rte_eal_get_configuration()->mem_config; | |||||
1345 | ||||||
1346 | /* hugetlbfs can be disabled */ | |||||
1347 | if (internal_config.no_hugetlbfs) { | |||||
1348 | struct rte_memseg_list *msl; | |||||
1349 | int n_segs, cur_seg, fd, flags; | |||||
1350 | #ifdef MEMFD_SUPPORTED | |||||
1351 | int memfd; | |||||
1352 | #endif | |||||
1353 | uint64_t page_sz; | |||||
1354 | ||||||
1355 | /* nohuge mode is legacy mode */ | |||||
1356 | internal_config.legacy_mem = 1; | |||||
1357 | ||||||
1358 | /* nohuge mode is single-file segments mode */ | |||||
1359 | internal_config.single_file_segments = 1; | |||||
1360 | ||||||
1361 | /* create a memseg list */ | |||||
1362 | msl = &mcfg->memsegs[0]; | |||||
1363 | ||||||
1364 | page_sz = RTE_PGSIZE_4K; | |||||
1365 | n_segs = internal_config.memory / page_sz; | |||||
1366 | ||||||
1367 | if (rte_fbarray_init(&msl->memseg_arr, "nohugemem", n_segs, | |||||
1368 | sizeof(struct rte_memseg))) { | |||||
1369 | RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list\n"); | |||||
1370 | return -1; | |||||
1371 | } | |||||
1372 | ||||||
1373 | /* set up parameters for anonymous mmap */ | |||||
1374 | fd = -1; | |||||
1375 | flags = MAP_PRIVATE0x02 | MAP_ANONYMOUS0x20; | |||||
1376 | ||||||
1377 | #ifdef MEMFD_SUPPORTED | |||||
1378 | /* create a memfd and store it in the segment fd table */ | |||||
1379 | memfd = memfd_create("nohuge", 0); | |||||
1380 | if (memfd < 0) { | |||||
1381 | RTE_LOG(DEBUG, EAL, "Cannot create memfd: %s\n",rte_log(8U, 0, "EAL" ": " "Cannot create memfd: %s\n", strerror ((*__errno_location ()))) | |||||
1382 | strerror(errno))rte_log(8U, 0, "EAL" ": " "Cannot create memfd: %s\n", strerror ((*__errno_location ()))); | |||||
1383 | RTE_LOG(DEBUG, EAL, "Falling back to anonymous map\n")rte_log(8U, 0, "EAL" ": " "Falling back to anonymous map\n"); | |||||
1384 | } else { | |||||
1385 | /* we got an fd - now resize it */ | |||||
1386 | if (ftruncate(memfd, internal_config.memory) < 0) { | |||||
1387 | RTE_LOG(ERR, EAL, "Cannot resize memfd: %s\n",rte_log(4U, 0, "EAL" ": " "Cannot resize memfd: %s\n", strerror ((*__errno_location ()))) | |||||
1388 | strerror(errno))rte_log(4U, 0, "EAL" ": " "Cannot resize memfd: %s\n", strerror ((*__errno_location ()))); | |||||
1389 | RTE_LOG(ERR, EAL, "Falling back to anonymous map\n")rte_log(4U, 0, "EAL" ": " "Falling back to anonymous map\n"); | |||||
1390 | close(memfd); | |||||
1391 | } else { | |||||
1392 | /* creating memfd-backed file was successful. | |||||
1393 | * we want changes to memfd to be visible to | |||||
1394 | * other processes (such as vhost backend), so | |||||
1395 | * map it as shared memory. | |||||
1396 | */ | |||||
1397 | RTE_LOG(DEBUG, EAL, "Using memfd for anonymous memory\n")rte_log(8U, 0, "EAL" ": " "Using memfd for anonymous memory\n" ); | |||||
1398 | fd = memfd; | |||||
1399 | flags = MAP_SHARED0x01; | |||||
1400 | } | |||||
1401 | } | |||||
1402 | #endif | |||||
1403 | addr = mmap(NULL((void*)0), internal_config.memory, PROT_READ0x1 | PROT_WRITE0x2, | |||||
1404 | flags, fd, 0); | |||||
1405 | if (addr == MAP_FAILED((void *) -1)) { | |||||
1406 | RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,rte_log(4U, 0, "EAL" ": " "%s: mmap() failed: %s\n", __func__ , strerror((*__errno_location ()))) | |||||
1407 | strerror(errno))rte_log(4U, 0, "EAL" ": " "%s: mmap() failed: %s\n", __func__ , strerror((*__errno_location ()))); | |||||
1408 | return -1; | |||||
1409 | } | |||||
1410 | msl->base_va = addr; | |||||
1411 | msl->page_sz = page_sz; | |||||
1412 | msl->socket_id = 0; | |||||
1413 | msl->len = internal_config.memory; | |||||
1414 | ||||||
1415 | /* we're in single-file segments mode, so only the segment list | |||||
1416 | * fd needs to be set up. | |||||
1417 | */ | |||||
1418 | if (fd != -1) { | |||||
1419 | if (eal_memalloc_set_seg_list_fd(0, fd) < 0) { | |||||
1420 | RTE_LOG(ERR, EAL, "Cannot set up segment list fd\n")rte_log(4U, 0, "EAL" ": " "Cannot set up segment list fd\n"); | |||||
1421 | /* not a serious error, proceed */ | |||||
1422 | } | |||||
1423 | } | |||||
1424 | ||||||
1425 | /* populate memsegs. each memseg is one page long */ | |||||
1426 | for (cur_seg = 0; cur_seg < n_segs; cur_seg++) { | |||||
1427 | arr = &msl->memseg_arr; | |||||
1428 | ||||||
1429 | ms = rte_fbarray_get(arr, cur_seg); | |||||
1430 | if (rte_eal_iova_mode() == RTE_IOVA_VA) | |||||
1431 | ms->iova = (uintptr_t)addr; | |||||
1432 | else | |||||
1433 | ms->iova = RTE_BAD_IOVA((rte_iova_t)-1); | |||||
1434 | ms->addr = addr; | |||||
1435 | ms->hugepage_sz = page_sz; | |||||
1436 | ms->socket_id = 0; | |||||
1437 | ms->len = page_sz; | |||||
1438 | ||||||
1439 | rte_fbarray_set_used(arr, cur_seg); | |||||
1440 | ||||||
1441 | addr = RTE_PTR_ADD(addr, (size_t)page_sz)((void*)((uintptr_t)(addr) + ((size_t)page_sz))); | |||||
1442 | } | |||||
1443 | if (mcfg->dma_maskbits && | |||||
1444 | rte_mem_check_dma_mask_thread_unsafe(mcfg->dma_maskbits)) { | |||||
1445 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n" , __func__) | |||||
1446 | "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n",rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n" , __func__) | |||||
1447 | __func__)rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n" , __func__); | |||||
1448 | if (rte_eal_iova_mode() == RTE_IOVA_VA && | |||||
1449 | rte_eal_using_phys_addrs()) | |||||
1450 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n" , __func__) | |||||
1451 | "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n",rte_log(4U, 0, "EAL" ": " "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n" , __func__) | |||||
1452 | __func__)rte_log(4U, 0, "EAL" ": " "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n" , __func__); | |||||
1453 | goto fail; | |||||
1454 | } | |||||
1455 | return 0; | |||||
1456 | } | |||||
1457 | ||||||
1458 | /* calculate total number of hugepages available. at this point we haven't | |||||
1459 | * yet started sorting them so they all are on socket 0 */ | |||||
1460 | for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) { | |||||
1461 | /* meanwhile, also initialize used_hp hugepage sizes in used_hp */ | |||||
1462 | used_hp[i].hugepage_sz = internal_config.hugepage_info[i].hugepage_sz; | |||||
1463 | ||||||
1464 | nr_hugepages += internal_config.hugepage_info[i].num_pages[0]; | |||||
1465 | } | |||||
1466 | ||||||
1467 | /* | |||||
1468 | * allocate a memory area for hugepage table. | |||||
1469 | * this isn't shared memory yet. due to the fact that we need some | |||||
1470 | * processing done on these pages, shared memory will be created | |||||
1471 | * at a later stage. | |||||
1472 | */ | |||||
1473 | tmp_hp = malloc(nr_hugepages * sizeof(struct hugepage_file)); | |||||
1474 | if (tmp_hp == NULL((void*)0)) | |||||
1475 | goto fail; | |||||
1476 | ||||||
1477 | memset(tmp_hp, 0, nr_hugepages * sizeof(struct hugepage_file)); | |||||
1478 | ||||||
1479 | hp_offset = 0; /* where we start the current page size entries */ | |||||
1480 | ||||||
1481 | huge_register_sigbus(); | |||||
1482 | ||||||
1483 | /* make a copy of socket_mem, needed for balanced allocation. */ | |||||
1484 | for (i = 0; i < RTE_MAX_NUMA_NODES8; i++) | |||||
1485 | memory[i] = internal_config.socket_mem[i]; | |||||
1486 | ||||||
1487 | /* map all hugepages and sort them */ | |||||
1488 | for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){ | |||||
1489 | unsigned pages_old, pages_new; | |||||
1490 | struct hugepage_info *hpi; | |||||
1491 | ||||||
1492 | /* | |||||
1493 | * we don't yet mark hugepages as used at this stage, so | |||||
1494 | * we just map all hugepages available to the system | |||||
1495 | * all hugepages are still located on socket 0 | |||||
1496 | */ | |||||
1497 | hpi = &internal_config.hugepage_info[i]; | |||||
1498 | ||||||
1499 | if (hpi->num_pages[0] == 0) | |||||
1500 | continue; | |||||
1501 | ||||||
1502 | /* map all hugepages available */ | |||||
1503 | pages_old = hpi->num_pages[0]; | |||||
1504 | pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, memory); | |||||
1505 | if (pages_new < pages_old) { | |||||
1506 | RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n" , pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000 )) | |||||
1507 | "%d not %d hugepages of size %u MB allocated\n",rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n" , pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000 )) | |||||
1508 | pages_new, pages_old,rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n" , pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000 )) | |||||
1509 | (unsigned)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n" , pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000 )); | |||||
1510 | ||||||
1511 | int pages = pages_old - pages_new; | |||||
1512 | ||||||
1513 | nr_hugepages -= pages; | |||||
1514 | hpi->num_pages[0] = pages_new; | |||||
1515 | if (pages_new == 0) | |||||
1516 | continue; | |||||
1517 | } | |||||
1518 | ||||||
1519 | if (phys_addrs_available && | |||||
1520 | rte_eal_iova_mode() != RTE_IOVA_VA) { | |||||
1521 | /* find physical addresses for each hugepage */ | |||||
1522 | if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0) { | |||||
1523 | RTE_LOG(DEBUG, EAL, "Failed to find phys addr "rte_log(8U, 0, "EAL" ": " "Failed to find phys addr " "for %u MB pages\n" , (unsigned int)(hpi->hugepage_sz / 0x100000)) | |||||
1524 | "for %u MB pages\n",rte_log(8U, 0, "EAL" ": " "Failed to find phys addr " "for %u MB pages\n" , (unsigned int)(hpi->hugepage_sz / 0x100000)) | |||||
1525 | (unsigned int)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "Failed to find phys addr " "for %u MB pages\n" , (unsigned int)(hpi->hugepage_sz / 0x100000)); | |||||
1526 | goto fail; | |||||
1527 | } | |||||
1528 | } else { | |||||
1529 | /* set physical addresses for each hugepage */ | |||||
1530 | if (set_physaddrs(&tmp_hp[hp_offset], hpi) < 0) { | |||||
1531 | RTE_LOG(DEBUG, EAL, "Failed to set phys addr "rte_log(8U, 0, "EAL" ": " "Failed to set phys addr " "for %u MB pages\n" , (unsigned int)(hpi->hugepage_sz / 0x100000)) | |||||
1532 | "for %u MB pages\n",rte_log(8U, 0, "EAL" ": " "Failed to set phys addr " "for %u MB pages\n" , (unsigned int)(hpi->hugepage_sz / 0x100000)) | |||||
1533 | (unsigned int)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "Failed to set phys addr " "for %u MB pages\n" , (unsigned int)(hpi->hugepage_sz / 0x100000)); | |||||
1534 | goto fail; | |||||
1535 | } | |||||
1536 | } | |||||
1537 | ||||||
1538 | if (find_numasocket(&tmp_hp[hp_offset], hpi) < 0){ | |||||
1539 | RTE_LOG(DEBUG, EAL, "Failed to find NUMA socket for %u MB pages\n",rte_log(8U, 0, "EAL" ": " "Failed to find NUMA socket for %u MB pages\n" , (unsigned)(hpi->hugepage_sz / 0x100000)) | |||||
1540 | (unsigned)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "Failed to find NUMA socket for %u MB pages\n" , (unsigned)(hpi->hugepage_sz / 0x100000)); | |||||
1541 | goto fail; | |||||
1542 | } | |||||
1543 | ||||||
1544 | qsort(&tmp_hp[hp_offset], hpi->num_pages[0], | |||||
1545 | sizeof(struct hugepage_file), cmp_physaddr); | |||||
1546 | ||||||
1547 | /* we have processed a num of hugepages of this size, so inc offset */ | |||||
1548 | hp_offset += hpi->num_pages[0]; | |||||
1549 | } | |||||
1550 | ||||||
1551 | huge_recover_sigbus(); | |||||
1552 | ||||||
1553 | if (internal_config.memory == 0 && internal_config.force_sockets == 0) | |||||
1554 | internal_config.memory = eal_get_hugepage_mem_size(); | |||||
1555 | ||||||
1556 | nr_hugefiles = nr_hugepages; | |||||
1557 | ||||||
1558 | ||||||
1559 | /* clean out the numbers of pages */ | |||||
1560 | for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) | |||||
1561 | for (j = 0; j < RTE_MAX_NUMA_NODES8; j++) | |||||
1562 | internal_config.hugepage_info[i].num_pages[j] = 0; | |||||
1563 | ||||||
1564 | /* get hugepages for each socket */ | |||||
1565 | for (i = 0; i < nr_hugefiles; i++) { | |||||
1566 | int socket = tmp_hp[i].socket_id; | |||||
1567 | ||||||
1568 | /* find a hugepage info with right size and increment num_pages */ | |||||
1569 | const int nb_hpsizes = RTE_MIN(MAX_HUGEPAGE_SIZES,__extension__ ({ __typeof__ (3) _a = (3); __typeof__ ((int)internal_config .num_hugepage_sizes) _b = ((int)internal_config.num_hugepage_sizes ); _a < _b ? _a : _b; }) | |||||
1570 | (int)internal_config.num_hugepage_sizes)__extension__ ({ __typeof__ (3) _a = (3); __typeof__ ((int)internal_config .num_hugepage_sizes) _b = ((int)internal_config.num_hugepage_sizes ); _a < _b ? _a : _b; }); | |||||
1571 | for (j = 0; j < nb_hpsizes; j++) { | |||||
1572 | if (tmp_hp[i].size == | |||||
1573 | internal_config.hugepage_info[j].hugepage_sz) { | |||||
1574 | internal_config.hugepage_info[j].num_pages[socket]++; | |||||
1575 | } | |||||
1576 | } | |||||
1577 | } | |||||
1578 | ||||||
1579 | /* make a copy of socket_mem, needed for number of pages calculation */ | |||||
1580 | for (i = 0; i < RTE_MAX_NUMA_NODES8; i++) | |||||
1581 | memory[i] = internal_config.socket_mem[i]; | |||||
1582 | ||||||
1583 | /* calculate final number of pages */ | |||||
1584 | nr_hugepages = calc_num_pages_per_socket(memory, | |||||
1585 | internal_config.hugepage_info, used_hp, | |||||
1586 | internal_config.num_hugepage_sizes); | |||||
1587 | ||||||
1588 | /* error if not enough memory available */ | |||||
1589 | if (nr_hugepages < 0) | |||||
1590 | goto fail; | |||||
1591 | ||||||
1592 | /* reporting in! */ | |||||
1593 | for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) { | |||||
1594 | for (j = 0; j < RTE_MAX_NUMA_NODES8; j++) { | |||||
1595 | if (used_hp[i].num_pages[j] > 0) { | |||||
1596 | RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j) | |||||
1597 | "Requesting %u pages of size %uMB"rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j) | |||||
1598 | " from socket %i\n",rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j) | |||||
1599 | used_hp[i].num_pages[j],rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j) | |||||
1600 | (unsigned)rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j) | |||||
1601 | (used_hp[i].hugepage_sz / 0x100000),rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j) | |||||
1602 | j)rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n" , used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz / 0x100000), j); | |||||
1603 | } | |||||
1604 | } | |||||
1605 | } | |||||
1606 | ||||||
1607 | /* create shared memory */ | |||||
1608 | hugepage = create_shared_memory(eal_hugepage_data_path(), | |||||
1609 | nr_hugefiles * sizeof(struct hugepage_file)); | |||||
1610 | ||||||
1611 | if (hugepage == NULL((void*)0)) { | |||||
1612 | RTE_LOG(ERR, EAL, "Failed to create shared memory!\n")rte_log(4U, 0, "EAL" ": " "Failed to create shared memory!\n" ); | |||||
1613 | goto fail; | |||||
1614 | } | |||||
1615 | memset(hugepage, 0, nr_hugefiles * sizeof(struct hugepage_file)); | |||||
1616 | ||||||
1617 | /* | |||||
1618 | * unmap pages that we won't need (looks at used_hp). | |||||
1619 | * also, sets final_va to NULL on pages that were unmapped. | |||||
1620 | */ | |||||
1621 | if (unmap_unneeded_hugepages(tmp_hp, used_hp, | |||||
1622 | internal_config.num_hugepage_sizes) < 0) { | |||||
1623 | RTE_LOG(ERR, EAL, "Unmapping and locking hugepages failed!\n")rte_log(4U, 0, "EAL" ": " "Unmapping and locking hugepages failed!\n" ); | |||||
1624 | goto fail; | |||||
1625 | } | |||||
1626 | ||||||
1627 | /* | |||||
1628 | * copy stuff from malloc'd hugepage* to the actual shared memory. | |||||
1629 | * this procedure only copies those hugepages that have orig_va | |||||
1630 | * not NULL. has overflow protection. | |||||
1631 | */ | |||||
1632 | if (copy_hugepages_to_shared_mem(hugepage, nr_hugefiles, | |||||
1633 | tmp_hp, nr_hugefiles) < 0) { | |||||
1634 | RTE_LOG(ERR, EAL, "Copying tables to shared memory failed!\n")rte_log(4U, 0, "EAL" ": " "Copying tables to shared memory failed!\n" ); | |||||
1635 | goto fail; | |||||
1636 | } | |||||
1637 | ||||||
1638 | #ifndef RTE_ARCH_641 | |||||
1639 | /* for legacy 32-bit mode, we did not preallocate VA space, so do it */ | |||||
1640 | if (internal_config.legacy_mem && | |||||
1641 | prealloc_segments(hugepage, nr_hugefiles)) { | |||||
1642 | RTE_LOG(ERR, EAL, "Could not preallocate VA space for hugepages\n")rte_log(4U, 0, "EAL" ": " "Could not preallocate VA space for hugepages\n" ); | |||||
1643 | goto fail; | |||||
1644 | } | |||||
1645 | #endif | |||||
1646 | ||||||
1647 | /* remap all pages we do need into memseg list VA space, so that those | |||||
1648 | * pages become first-class citizens in DPDK memory subsystem | |||||
1649 | */ | |||||
1650 | if (remap_needed_hugepages(hugepage, nr_hugefiles)) { | |||||
1651 | RTE_LOG(ERR, EAL, "Couldn't remap hugepage files into memseg lists\n")rte_log(4U, 0, "EAL" ": " "Couldn't remap hugepage files into memseg lists\n" ); | |||||
1652 | goto fail; | |||||
1653 | } | |||||
1654 | ||||||
1655 | /* free the hugepage backing files */ | |||||
1656 | if (internal_config.hugepage_unlink && | |||||
1657 | unlink_hugepage_files(tmp_hp, internal_config.num_hugepage_sizes) < 0) { | |||||
1658 | RTE_LOG(ERR, EAL, "Unlinking hugepage files failed!\n")rte_log(4U, 0, "EAL" ": " "Unlinking hugepage files failed!\n" ); | |||||
1659 | goto fail; | |||||
1660 | } | |||||
1661 | ||||||
1662 | /* free the temporary hugepage table */ | |||||
1663 | free(tmp_hp); | |||||
1664 | tmp_hp = NULL((void*)0); | |||||
1665 | ||||||
1666 | munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file)); | |||||
1667 | hugepage = NULL((void*)0); | |||||
1668 | ||||||
1669 | /* we're not going to allocate more pages, so release VA space for | |||||
1670 | * unused memseg lists | |||||
1671 | */ | |||||
1672 | for (i = 0; i < RTE_MAX_MEMSEG_LISTS64; i++) { | |||||
1673 | struct rte_memseg_list *msl = &mcfg->memsegs[i]; | |||||
1674 | size_t mem_sz; | |||||
1675 | ||||||
1676 | /* skip inactive lists */ | |||||
1677 | if (msl->base_va == NULL((void*)0)) | |||||
1678 | continue; | |||||
1679 | /* skip lists where there is at least one page allocated */ | |||||
1680 | if (msl->memseg_arr.count > 0) | |||||
1681 | continue; | |||||
1682 | /* this is an unused list, deallocate it */ | |||||
1683 | mem_sz = msl->len; | |||||
1684 | munmap(msl->base_va, mem_sz); | |||||
1685 | msl->base_va = NULL((void*)0); | |||||
1686 | ||||||
1687 | /* destroy backing fbarray */ | |||||
1688 | rte_fbarray_destroy(&msl->memseg_arr); | |||||
1689 | } | |||||
1690 | ||||||
1691 | if (mcfg->dma_maskbits && | |||||
1692 | rte_mem_check_dma_mask_thread_unsafe(mcfg->dma_maskbits)) { | |||||
1693 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n" , __func__) | |||||
1694 | "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n",rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n" , __func__) | |||||
1695 | __func__)rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n" , __func__); | |||||
1696 | goto fail; | |||||
1697 | } | |||||
1698 | ||||||
1699 | return 0; | |||||
1700 | ||||||
1701 | fail: | |||||
1702 | huge_recover_sigbus(); | |||||
1703 | free(tmp_hp); | |||||
1704 | if (hugepage != NULL((void*)0)) | |||||
1705 | munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file)); | |||||
1706 | ||||||
1707 | return -1; | |||||
1708 | } | |||||
1709 | ||||||
1710 | static int __rte_unused__attribute__((__unused__)) | |||||
1711 | hugepage_count_walk(const struct rte_memseg_list *msl, void *arg) | |||||
1712 | { | |||||
1713 | struct hugepage_info *hpi = arg; | |||||
1714 | ||||||
1715 | if (msl->page_sz != hpi->hugepage_sz) | |||||
1716 | return 0; | |||||
1717 | ||||||
1718 | hpi->num_pages[msl->socket_id] += msl->memseg_arr.len; | |||||
1719 | return 0; | |||||
1720 | } | |||||
1721 | ||||||
1722 | static int | |||||
1723 | limits_callback(int socket_id, size_t cur_limit, size_t new_len) | |||||
1724 | { | |||||
1725 | RTE_SET_USED(socket_id)(void)(socket_id); | |||||
1726 | RTE_SET_USED(cur_limit)(void)(cur_limit); | |||||
1727 | RTE_SET_USED(new_len)(void)(new_len); | |||||
1728 | return -1; | |||||
1729 | } | |||||
1730 | ||||||
1731 | static int | |||||
1732 | eal_hugepage_init(void) | |||||
1733 | { | |||||
1734 | struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES3]; | |||||
1735 | uint64_t memory[RTE_MAX_NUMA_NODES8]; | |||||
1736 | int hp_sz_idx, socket_id; | |||||
1737 | ||||||
1738 | test_phys_addrs_available(); | |||||
1739 | ||||||
1740 | memset(used_hp, 0, sizeof(used_hp)); | |||||
1741 | ||||||
1742 | for (hp_sz_idx = 0; | |||||
1743 | hp_sz_idx < (int) internal_config.num_hugepage_sizes; | |||||
1744 | hp_sz_idx++) { | |||||
1745 | #ifndef RTE_ARCH_641 | |||||
1746 | struct hugepage_info dummy; | |||||
1747 | unsigned int i; | |||||
1748 | #endif | |||||
1749 | /* also initialize used_hp hugepage sizes in used_hp */ | |||||
1750 | struct hugepage_info *hpi; | |||||
1751 | hpi = &internal_config.hugepage_info[hp_sz_idx]; | |||||
1752 | used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz; | |||||
1753 | ||||||
1754 | #ifndef RTE_ARCH_641 | |||||
1755 | /* for 32-bit, limit number of pages on socket to whatever we've | |||||
1756 | * preallocated, as we cannot allocate more. | |||||
1757 | */ | |||||
1758 | memset(&dummy, 0, sizeof(dummy)); | |||||
1759 | dummy.hugepage_sz = hpi->hugepage_sz; | |||||
1760 | if (rte_memseg_list_walk(hugepage_count_walk, &dummy) < 0) | |||||
1761 | return -1; | |||||
1762 | ||||||
1763 | for (i = 0; i < RTE_DIM(dummy.num_pages)(sizeof (dummy.num_pages) / sizeof ((dummy.num_pages)[0])); i++) { | |||||
1764 | hpi->num_pages[i] = RTE_MIN(hpi->num_pages[i],__extension__ ({ __typeof__ (hpi->num_pages[i]) _a = (hpi-> num_pages[i]); __typeof__ (dummy.num_pages[i]) _b = (dummy.num_pages [i]); _a < _b ? _a : _b; }) | |||||
1765 | dummy.num_pages[i])__extension__ ({ __typeof__ (hpi->num_pages[i]) _a = (hpi-> num_pages[i]); __typeof__ (dummy.num_pages[i]) _b = (dummy.num_pages [i]); _a < _b ? _a : _b; }); | |||||
1766 | } | |||||
1767 | #endif | |||||
1768 | } | |||||
1769 | ||||||
1770 | /* make a copy of socket_mem, needed for balanced allocation. */ | |||||
1771 | for (hp_sz_idx = 0; hp_sz_idx < RTE_MAX_NUMA_NODES8; hp_sz_idx++) | |||||
1772 | memory[hp_sz_idx] = internal_config.socket_mem[hp_sz_idx]; | |||||
1773 | ||||||
1774 | /* calculate final number of pages */ | |||||
1775 | if (calc_num_pages_per_socket(memory, | |||||
1776 | internal_config.hugepage_info, used_hp, | |||||
1777 | internal_config.num_hugepage_sizes) < 0) | |||||
1778 | return -1; | |||||
1779 | ||||||
1780 | for (hp_sz_idx = 0; | |||||
1781 | hp_sz_idx < (int)internal_config.num_hugepage_sizes; | |||||
1782 | hp_sz_idx++) { | |||||
1783 | for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES8; | |||||
1784 | socket_id++) { | |||||
1785 | struct rte_memseg **pages; | |||||
1786 | struct hugepage_info *hpi = &used_hp[hp_sz_idx]; | |||||
1787 | unsigned int num_pages = hpi->num_pages[socket_id]; | |||||
1788 | unsigned int num_pages_alloc; | |||||
1789 | ||||||
1790 | if (num_pages == 0) | |||||
1791 | continue; | |||||
1792 | ||||||
1793 | RTE_LOG(DEBUG, EAL, "Allocating %u pages of size %" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Allocating %u pages of size %" "l" "u" "M on socket %i\n", num_pages, hpi->hugepage_sz >> 20, socket_id) | |||||
1794 | num_pages, hpi->hugepage_sz >> 20, socket_id)rte_log(8U, 0, "EAL" ": " "Allocating %u pages of size %" "l" "u" "M on socket %i\n", num_pages, hpi->hugepage_sz >> 20, socket_id); | |||||
1795 | ||||||
1796 | /* we may not be able to allocate all pages in one go, | |||||
1797 | * because we break up our memory map into multiple | |||||
1798 | * memseg lists. therefore, try allocating multiple | |||||
1799 | * times and see if we can get the desired number of | |||||
1800 | * pages from multiple allocations. | |||||
1801 | */ | |||||
1802 | ||||||
1803 | num_pages_alloc = 0; | |||||
1804 | do { | |||||
1805 | int i, cur_pages, needed; | |||||
1806 | ||||||
1807 | needed = num_pages - num_pages_alloc; | |||||
1808 | ||||||
1809 | pages = malloc(sizeof(*pages) * needed); | |||||
1810 | ||||||
1811 | /* do not request exact number of pages */ | |||||
1812 | cur_pages = eal_memalloc_alloc_seg_bulk(pages, | |||||
1813 | needed, hpi->hugepage_sz, | |||||
1814 | socket_id, false0); | |||||
1815 | if (cur_pages <= 0) { | |||||
1816 | free(pages); | |||||
1817 | return -1; | |||||
1818 | } | |||||
1819 | ||||||
1820 | /* mark preallocated pages as unfreeable */ | |||||
1821 | for (i = 0; i < cur_pages; i++) { | |||||
1822 | struct rte_memseg *ms = pages[i]; | |||||
1823 | ms->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE(1 << 0); | |||||
1824 | } | |||||
1825 | free(pages); | |||||
1826 | ||||||
1827 | num_pages_alloc += cur_pages; | |||||
1828 | } while (num_pages_alloc != num_pages); | |||||
1829 | } | |||||
1830 | } | |||||
1831 | /* if socket limits were specified, set them */ | |||||
1832 | if (internal_config.force_socket_limits) { | |||||
1833 | unsigned int i; | |||||
1834 | for (i = 0; i < RTE_MAX_NUMA_NODES8; i++) { | |||||
1835 | uint64_t limit = internal_config.socket_limit[i]; | |||||
1836 | if (limit == 0) | |||||
1837 | continue; | |||||
1838 | if (rte_mem_alloc_validator_register("socket-limit", | |||||
1839 | limits_callback, i, limit)) | |||||
1840 | RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n")rte_log(4U, 0, "EAL" ": " "Failed to register socket limits validator callback\n" ); | |||||
1841 | } | |||||
1842 | } | |||||
1843 | return 0; | |||||
1844 | } | |||||
1845 | ||||||
1846 | /* | |||||
1847 | * uses fstat to report the size of a file on disk | |||||
1848 | */ | |||||
1849 | static off_t | |||||
1850 | getFileSize(int fd) | |||||
1851 | { | |||||
1852 | struct stat st; | |||||
1853 | if (fstat(fd, &st) < 0) | |||||
1854 | return 0; | |||||
1855 | return st.st_size; | |||||
1856 | } | |||||
1857 | ||||||
1858 | /* | |||||
1859 | * This creates the memory mappings in the secondary process to match that of | |||||
1860 | * the server process. It goes through each memory segment in the DPDK runtime | |||||
1861 | * configuration and finds the hugepages which form that segment, mapping them | |||||
1862 | * in order to form a contiguous block in the virtual memory space | |||||
1863 | */ | |||||
1864 | static int | |||||
1865 | eal_legacy_hugepage_attach(void) | |||||
1866 | { | |||||
1867 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |||||
1868 | struct hugepage_file *hp = NULL((void*)0); | |||||
1869 | unsigned int num_hp = 0; | |||||
1870 | unsigned int i = 0; | |||||
1871 | unsigned int cur_seg; | |||||
1872 | off_t size = 0; | |||||
1873 | int fd, fd_hugepage = -1; | |||||
1874 | ||||||
1875 | if (aslr_enabled() > 0) { | |||||
1876 | RTE_LOG(WARNING, EAL, "WARNING: Address Space Layout Randomization "rte_log(5U, 0, "EAL" ": " "WARNING: Address Space Layout Randomization " "(ASLR) is enabled in the kernel.\n") | |||||
1877 | "(ASLR) is enabled in the kernel.\n")rte_log(5U, 0, "EAL" ": " "WARNING: Address Space Layout Randomization " "(ASLR) is enabled in the kernel.\n"); | |||||
1878 | RTE_LOG(WARNING, EAL, " This may cause issues with mapping memory "rte_log(5U, 0, "EAL" ": " " This may cause issues with mapping memory " "into secondary processes\n") | |||||
1879 | "into secondary processes\n")rte_log(5U, 0, "EAL" ": " " This may cause issues with mapping memory " "into secondary processes\n"); | |||||
1880 | } | |||||
1881 | ||||||
1882 | test_phys_addrs_available(); | |||||
1883 | ||||||
1884 | fd_hugepage = open(eal_hugepage_data_path(), O_RDONLY00); | |||||
1885 | if (fd_hugepage < 0) { | |||||
1886 | RTE_LOG(ERR, EAL, "Could not open %s\n",rte_log(4U, 0, "EAL" ": " "Could not open %s\n", eal_hugepage_data_path ()) | |||||
1887 | eal_hugepage_data_path())rte_log(4U, 0, "EAL" ": " "Could not open %s\n", eal_hugepage_data_path ()); | |||||
1888 | goto error; | |||||
1889 | } | |||||
1890 | ||||||
1891 | size = getFileSize(fd_hugepage); | |||||
1892 | hp = mmap(NULL((void*)0), size, PROT_READ0x1, MAP_PRIVATE0x02, fd_hugepage, 0); | |||||
1893 | if (hp == MAP_FAILED((void *) -1)) { | |||||
1894 | RTE_LOG(ERR, EAL, "Could not mmap %s\n",rte_log(4U, 0, "EAL" ": " "Could not mmap %s\n", eal_hugepage_data_path ()) | |||||
1895 | eal_hugepage_data_path())rte_log(4U, 0, "EAL" ": " "Could not mmap %s\n", eal_hugepage_data_path ()); | |||||
1896 | goto error; | |||||
1897 | } | |||||
1898 | ||||||
1899 | num_hp = size / sizeof(struct hugepage_file); | |||||
1900 | RTE_LOG(DEBUG, EAL, "Analysing %u files\n", num_hp)rte_log(8U, 0, "EAL" ": " "Analysing %u files\n", num_hp); | |||||
1901 | ||||||
1902 | /* map all segments into memory to make sure we get the addrs. the | |||||
1903 | * segments themselves are already in memseg list (which is shared and | |||||
1904 | * has its VA space already preallocated), so we just need to map | |||||
1905 | * everything into correct addresses. | |||||
1906 | */ | |||||
1907 | for (i = 0; i < num_hp; i++) { | |||||
1908 | struct hugepage_file *hf = &hp[i]; | |||||
1909 | size_t map_sz = hf->size; | |||||
1910 | void *map_addr = hf->final_va; | |||||
1911 | int msl_idx, ms_idx; | |||||
1912 | struct rte_memseg_list *msl; | |||||
1913 | struct rte_memseg *ms; | |||||
1914 | ||||||
1915 | /* if size is zero, no more pages left */ | |||||
1916 | if (map_sz == 0) | |||||
1917 | break; | |||||
1918 | ||||||
1919 | fd = open(hf->filepath, O_RDWR02); | |||||
1920 | if (fd < 0) { | |||||
1921 | RTE_LOG(ERR, EAL, "Could not open %s: %s\n",rte_log(4U, 0, "EAL" ": " "Could not open %s: %s\n", hf->filepath , strerror((*__errno_location ()))) | |||||
1922 | hf->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Could not open %s: %s\n", hf->filepath , strerror((*__errno_location ()))); | |||||
1923 | goto error; | |||||
1924 | } | |||||
1925 | ||||||
1926 | map_addr = mmap(map_addr, map_sz, PROT_READ0x1 | PROT_WRITE0x2, | |||||
1927 | MAP_SHARED0x01 | MAP_FIXED0x10, fd, 0); | |||||
1928 | if (map_addr == MAP_FAILED((void *) -1)) { | |||||
1929 | RTE_LOG(ERR, EAL, "Could not map %s: %s\n",rte_log(4U, 0, "EAL" ": " "Could not map %s: %s\n", hf->filepath , strerror((*__errno_location ()))) | |||||
1930 | hf->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Could not map %s: %s\n", hf->filepath , strerror((*__errno_location ()))); | |||||
1931 | goto fd_error; | |||||
1932 | } | |||||
1933 | ||||||
1934 | /* set shared lock on the file. */ | |||||
1935 | if (flock(fd, LOCK_SH1) < 0) { | |||||
1936 | RTE_LOG(DEBUG, EAL, "%s(): Locking file failed: %s\n",rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed: %s\n", __func__ , strerror((*__errno_location ()))) | |||||
1937 | __func__, strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed: %s\n", __func__ , strerror((*__errno_location ()))); | |||||
1938 | goto fd_error; | |||||
1939 | } | |||||
1940 | ||||||
1941 | /* find segment data */ | |||||
1942 | msl = rte_mem_virt2memseg_list(map_addr); | |||||
1943 | if (msl == NULL((void*)0)) { | |||||
1944 | RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg list\n",rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg list\n", __func__ ) | |||||
1945 | __func__)rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg list\n", __func__ ); | |||||
1946 | goto fd_error; | |||||
1947 | } | |||||
1948 | ms = rte_mem_virt2memseg(map_addr, msl); | |||||
1949 | if (ms == NULL((void*)0)) { | |||||
1950 | RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg\n",rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg\n", __func__ ) | |||||
1951 | __func__)rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg\n", __func__ ); | |||||
1952 | goto fd_error; | |||||
1953 | } | |||||
1954 | ||||||
1955 | msl_idx = msl - mcfg->memsegs; | |||||
1956 | ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms); | |||||
1957 | if (ms_idx < 0) { | |||||
1958 | RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg idx\n",rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg idx\n", __func__ ) | |||||
1959 | __func__)rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg idx\n", __func__ ); | |||||
1960 | goto fd_error; | |||||
1961 | } | |||||
1962 | ||||||
1963 | /* store segment fd internally */ | |||||
1964 | if (eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd) < 0) | |||||
1965 | RTE_LOG(ERR, EAL, "Could not store segment fd: %s\n",rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n", rte_strerror((per_lcore__rte_errno))) | |||||
1966 | rte_strerror(rte_errno))rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n", rte_strerror((per_lcore__rte_errno))); | |||||
1967 | } | |||||
1968 | /* unmap the hugepage config file, since we are done using it */ | |||||
1969 | munmap(hp, size); | |||||
1970 | close(fd_hugepage); | |||||
1971 | return 0; | |||||
1972 | ||||||
1973 | fd_error: | |||||
1974 | close(fd); | |||||
1975 | error: | |||||
1976 | /* map all segments into memory to make sure we get the addrs */ | |||||
1977 | cur_seg = 0; | |||||
1978 | for (cur_seg = 0; cur_seg < i; cur_seg++) { | |||||
1979 | struct hugepage_file *hf = &hp[i]; | |||||
1980 | size_t map_sz = hf->size; | |||||
1981 | void *map_addr = hf->final_va; | |||||
1982 | ||||||
1983 | munmap(map_addr, map_sz); | |||||
1984 | } | |||||
1985 | if (hp != NULL((void*)0) && hp != MAP_FAILED((void *) -1)) | |||||
1986 | munmap(hp, size); | |||||
1987 | if (fd_hugepage >= 0) | |||||
1988 | close(fd_hugepage); | |||||
1989 | return -1; | |||||
1990 | } | |||||
1991 | ||||||
1992 | static int | |||||
1993 | eal_hugepage_attach(void) | |||||
1994 | { | |||||
1995 | if (eal_memalloc_sync_with_primary()) { | |||||
1996 | RTE_LOG(ERR, EAL, "Could not map memory from primary process\n")rte_log(4U, 0, "EAL" ": " "Could not map memory from primary process\n" ); | |||||
1997 | if (aslr_enabled() > 0) | |||||
1998 | RTE_LOG(ERR, EAL, "It is recommended to disable ASLR in the kernel and retry running both primary and secondary processes\n")rte_log(4U, 0, "EAL" ": " "It is recommended to disable ASLR in the kernel and retry running both primary and secondary processes\n" ); | |||||
1999 | return -1; | |||||
2000 | } | |||||
2001 | return 0; | |||||
2002 | } | |||||
2003 | ||||||
2004 | int | |||||
2005 | rte_eal_hugepage_init(void) | |||||
2006 | { | |||||
2007 | return internal_config.legacy_mem ? | |||||
2008 | eal_legacy_hugepage_init() : | |||||
2009 | eal_hugepage_init(); | |||||
2010 | } | |||||
2011 | ||||||
2012 | int | |||||
2013 | rte_eal_hugepage_attach(void) | |||||
2014 | { | |||||
2015 | return internal_config.legacy_mem ? | |||||
2016 | eal_legacy_hugepage_attach() : | |||||
2017 | eal_hugepage_attach(); | |||||
2018 | } | |||||
2019 | ||||||
2020 | int | |||||
2021 | rte_eal_using_phys_addrs(void) | |||||
2022 | { | |||||
2023 | return phys_addrs_available; | |||||
2024 | } | |||||
2025 | ||||||
2026 | static int __rte_unused__attribute__((__unused__)) | |||||
2027 | memseg_primary_init_32(void) | |||||
2028 | { | |||||
2029 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |||||
2030 | int active_sockets, hpi_idx, msl_idx = 0; | |||||
2031 | unsigned int socket_id, i; | |||||
2032 | struct rte_memseg_list *msl; | |||||
2033 | uint64_t extra_mem_per_socket, total_extra_mem, total_requested_mem; | |||||
2034 | uint64_t max_mem; | |||||
2035 | ||||||
2036 | /* no-huge does not need this at all */ | |||||
2037 | if (internal_config.no_hugetlbfs) | |||||
2038 | return 0; | |||||
2039 | ||||||
2040 | /* this is a giant hack, but desperate times call for desperate | |||||
2041 | * measures. in legacy 32-bit mode, we cannot preallocate VA space, | |||||
2042 | * because having upwards of 2 gigabytes of VA space already mapped will | |||||
2043 | * interfere with our ability to map and sort hugepages. | |||||
2044 | * | |||||
2045 | * therefore, in legacy 32-bit mode, we will be initializing memseg | |||||
2046 | * lists much later - in eal_memory.c, right after we unmap all the | |||||
2047 | * unneeded pages. this will not affect secondary processes, as those | |||||
2048 | * should be able to mmap the space without (too many) problems. | |||||
2049 | */ | |||||
2050 | if (internal_config.legacy_mem) | |||||
2051 | return 0; | |||||
2052 | ||||||
2053 | /* 32-bit mode is a very special case. we cannot know in advance where | |||||
2054 | * the user will want to allocate their memory, so we have to do some | |||||
2055 | * heuristics. | |||||
2056 | */ | |||||
2057 | active_sockets = 0; | |||||
2058 | total_requested_mem = 0; | |||||
2059 | if (internal_config.force_sockets) | |||||
2060 | for (i = 0; i < rte_socket_count(); i++) { | |||||
2061 | uint64_t mem; | |||||
2062 | ||||||
2063 | socket_id = rte_socket_id_by_idx(i); | |||||
2064 | mem = internal_config.socket_mem[socket_id]; | |||||
2065 | ||||||
2066 | if (mem == 0) | |||||
2067 | continue; | |||||
2068 | ||||||
2069 | active_sockets++; | |||||
2070 | total_requested_mem += mem; | |||||
2071 | } | |||||
2072 | else | |||||
2073 | total_requested_mem = internal_config.memory; | |||||
2074 | ||||||
2075 | max_mem = (uint64_t)RTE_MAX_MEM_MB524288 << 20; | |||||
2076 | if (total_requested_mem > max_mem) { | |||||
2077 | RTE_LOG(ERR, EAL, "Invalid parameters: 32-bit process can at most use %uM of memory\n",rte_log(4U, 0, "EAL" ": " "Invalid parameters: 32-bit process can at most use %uM of memory\n" , (unsigned int)(max_mem >> 20)) | |||||
2078 | (unsigned int)(max_mem >> 20))rte_log(4U, 0, "EAL" ": " "Invalid parameters: 32-bit process can at most use %uM of memory\n" , (unsigned int)(max_mem >> 20)); | |||||
2079 | return -1; | |||||
2080 | } | |||||
2081 | total_extra_mem = max_mem - total_requested_mem; | |||||
2082 | extra_mem_per_socket = active_sockets == 0 ? total_extra_mem : | |||||
2083 | total_extra_mem / active_sockets; | |||||
2084 | ||||||
2085 | /* the allocation logic is a little bit convoluted, but here's how it | |||||
2086 | * works, in a nutshell: | |||||
2087 | * - if user hasn't specified on which sockets to allocate memory via | |||||
2088 | * --socket-mem, we allocate all of our memory on master core socket. | |||||
2089 | * - if user has specified sockets to allocate memory on, there may be | |||||
2090 | * some "unused" memory left (e.g. if user has specified --socket-mem | |||||
2091 | * such that not all memory adds up to 2 gigabytes), so add it to all | |||||
2092 | * sockets that are in use equally. | |||||
2093 | * | |||||
2094 | * page sizes are sorted by size in descending order, so we can safely | |||||
2095 | * assume that we dispense with bigger page sizes first. | |||||
2096 | */ | |||||
2097 | ||||||
2098 | /* create memseg lists */ | |||||
2099 | for (i = 0; i < rte_socket_count(); i++) { | |||||
2100 | int hp_sizes = (int) internal_config.num_hugepage_sizes; | |||||
2101 | uint64_t max_socket_mem, cur_socket_mem; | |||||
2102 | unsigned int master_lcore_socket; | |||||
2103 | struct rte_config *cfg = rte_eal_get_configuration(); | |||||
2104 | bool_Bool skip; | |||||
2105 | ||||||
2106 | socket_id = rte_socket_id_by_idx(i); | |||||
2107 | ||||||
2108 | #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
2109 | /* we can still sort pages by socket in legacy mode */ | |||||
2110 | if (!internal_config.legacy_mem && socket_id > 0) | |||||
2111 | break; | |||||
2112 | #endif | |||||
2113 | ||||||
2114 | /* if we didn't specifically request memory on this socket */ | |||||
2115 | skip = active_sockets != 0 && | |||||
2116 | internal_config.socket_mem[socket_id] == 0; | |||||
2117 | /* ...or if we didn't specifically request memory on *any* | |||||
2118 | * socket, and this is not master lcore | |||||
2119 | */ | |||||
2120 | master_lcore_socket = rte_lcore_to_socket_id(cfg->master_lcore); | |||||
2121 | skip |= active_sockets == 0 && socket_id != master_lcore_socket; | |||||
2122 | ||||||
2123 | if (skip) { | |||||
2124 | RTE_LOG(DEBUG, EAL, "Will not preallocate memory on socket %u\n",rte_log(8U, 0, "EAL" ": " "Will not preallocate memory on socket %u\n" , socket_id) | |||||
2125 | socket_id)rte_log(8U, 0, "EAL" ": " "Will not preallocate memory on socket %u\n" , socket_id); | |||||
2126 | continue; | |||||
2127 | } | |||||
2128 | ||||||
2129 | /* max amount of memory on this socket */ | |||||
2130 | max_socket_mem = (active_sockets != 0 ? | |||||
2131 | internal_config.socket_mem[socket_id] : | |||||
2132 | internal_config.memory) + | |||||
2133 | extra_mem_per_socket; | |||||
2134 | cur_socket_mem = 0; | |||||
2135 | ||||||
2136 | for (hpi_idx = 0; hpi_idx < hp_sizes; hpi_idx++) { | |||||
2137 | uint64_t max_pagesz_mem, cur_pagesz_mem = 0; | |||||
2138 | uint64_t hugepage_sz; | |||||
2139 | struct hugepage_info *hpi; | |||||
2140 | int type_msl_idx, max_segs, total_segs = 0; | |||||
2141 | ||||||
2142 | hpi = &internal_config.hugepage_info[hpi_idx]; | |||||
2143 | hugepage_sz = hpi->hugepage_sz; | |||||
2144 | ||||||
2145 | /* check if pages are actually available */ | |||||
2146 | if (hpi->num_pages[socket_id] == 0) | |||||
2147 | continue; | |||||
2148 | ||||||
2149 | max_segs = RTE_MAX_MEMSEG_PER_TYPE32768; | |||||
2150 | max_pagesz_mem = max_socket_mem - cur_socket_mem; | |||||
2151 | ||||||
2152 | /* make it multiple of page size */ | |||||
2153 | max_pagesz_mem = RTE_ALIGN_FLOOR(max_pagesz_mem,(__typeof__(max_pagesz_mem))((max_pagesz_mem) & (~((__typeof__ (max_pagesz_mem))((hugepage_sz) - 1)))) | |||||
2154 | hugepage_sz)(__typeof__(max_pagesz_mem))((max_pagesz_mem) & (~((__typeof__ (max_pagesz_mem))((hugepage_sz) - 1)))); | |||||
2155 | ||||||
2156 | RTE_LOG(DEBUG, EAL, "Attempting to preallocate "rte_log(8U, 0, "EAL" ": " "Attempting to preallocate " "%" "l" "u" "M on socket %i\n", max_pagesz_mem >> 20, socket_id ) | |||||
2157 | "%" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Attempting to preallocate " "%" "l" "u" "M on socket %i\n", max_pagesz_mem >> 20, socket_id ) | |||||
2158 | max_pagesz_mem >> 20, socket_id)rte_log(8U, 0, "EAL" ": " "Attempting to preallocate " "%" "l" "u" "M on socket %i\n", max_pagesz_mem >> 20, socket_id ); | |||||
2159 | ||||||
2160 | type_msl_idx = 0; | |||||
2161 | while (cur_pagesz_mem < max_pagesz_mem && | |||||
2162 | total_segs < max_segs) { | |||||
2163 | uint64_t cur_mem; | |||||
2164 | unsigned int n_segs; | |||||
2165 | ||||||
2166 | if (msl_idx >= RTE_MAX_MEMSEG_LISTS64) { | |||||
2167 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS") | |||||
2168 | "No more space in memseg lists, please increase %s\n",rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS") | |||||
2169 | RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS"); | |||||
2170 | return -1; | |||||
2171 | } | |||||
2172 | ||||||
2173 | msl = &mcfg->memsegs[msl_idx]; | |||||
2174 | ||||||
2175 | cur_mem = get_mem_amount(hugepage_sz, | |||||
2176 | max_pagesz_mem); | |||||
2177 | n_segs = cur_mem / hugepage_sz; | |||||
2178 | ||||||
2179 | if (alloc_memseg_list(msl, hugepage_sz, n_segs, | |||||
2180 | socket_id, type_msl_idx)) { | |||||
2181 | /* failing to allocate a memseg list is | |||||
2182 | * a serious error. | |||||
2183 | */ | |||||
2184 | RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list\n"); | |||||
2185 | return -1; | |||||
2186 | } | |||||
2187 | ||||||
2188 | if (alloc_va_space(msl)) { | |||||
2189 | /* if we couldn't allocate VA space, we | |||||
2190 | * can try with smaller page sizes. | |||||
2191 | */ | |||||
2192 | RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list, retrying with different page size\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space for memseg list, retrying with different page size\n" ); | |||||
2193 | /* deallocate memseg list */ | |||||
2194 | if (free_memseg_list(msl)) | |||||
2195 | return -1; | |||||
2196 | break; | |||||
2197 | } | |||||
2198 | ||||||
2199 | total_segs += msl->memseg_arr.len; | |||||
2200 | cur_pagesz_mem = total_segs * hugepage_sz; | |||||
2201 | type_msl_idx++; | |||||
2202 | msl_idx++; | |||||
2203 | } | |||||
2204 | cur_socket_mem += cur_pagesz_mem; | |||||
2205 | } | |||||
2206 | if (cur_socket_mem == 0) { | |||||
2207 | RTE_LOG(ERR, EAL, "Cannot allocate VA space on socket %u\n",rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space on socket %u\n" , socket_id) | |||||
2208 | socket_id)rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space on socket %u\n" , socket_id); | |||||
2209 | return -1; | |||||
2210 | } | |||||
2211 | } | |||||
2212 | ||||||
2213 | return 0; | |||||
2214 | } | |||||
2215 | ||||||
2216 | static int __rte_unused__attribute__((__unused__)) | |||||
2217 | memseg_primary_init(void) | |||||
2218 | { | |||||
2219 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |||||
2220 | struct memtype { | |||||
2221 | uint64_t page_sz; | |||||
2222 | int socket_id; | |||||
2223 | } *memtypes = NULL((void*)0); | |||||
2224 | int i, hpi_idx, msl_idx, ret = -1; /* fail unless told to succeed */ | |||||
2225 | struct rte_memseg_list *msl; | |||||
2226 | uint64_t max_mem, max_mem_per_type; | |||||
2227 | unsigned int max_seglists_per_type; | |||||
2228 | unsigned int n_memtypes, cur_type; | |||||
2229 | ||||||
2230 | /* no-huge does not need this at all */ | |||||
2231 | if (internal_config.no_hugetlbfs) | |||||
2232 | return 0; | |||||
2233 | ||||||
2234 | /* | |||||
2235 | * figuring out amount of memory we're going to have is a long and very | |||||
2236 | * involved process. the basic element we're operating with is a memory | |||||
2237 | * type, defined as a combination of NUMA node ID and page size (so that | |||||
2238 | * e.g. 2 sockets with 2 page sizes yield 4 memory types in total). | |||||
2239 | * | |||||
2240 | * deciding amount of memory going towards each memory type is a | |||||
2241 | * balancing act between maximum segments per type, maximum memory per | |||||
2242 | * type, and number of detected NUMA nodes. the goal is to make sure | |||||
2243 | * each memory type gets at least one memseg list. | |||||
2244 | * | |||||
2245 | * the total amount of memory is limited by RTE_MAX_MEM_MB value. | |||||
2246 | * | |||||
2247 | * the total amount of memory per type is limited by either | |||||
2248 | * RTE_MAX_MEM_MB_PER_TYPE, or by RTE_MAX_MEM_MB divided by the number | |||||
2249 | * of detected NUMA nodes. additionally, maximum number of segments per | |||||
2250 | * type is also limited by RTE_MAX_MEMSEG_PER_TYPE. this is because for | |||||
2251 | * smaller page sizes, it can take hundreds of thousands of segments to | |||||
2252 | * reach the above specified per-type memory limits. | |||||
2253 | * | |||||
2254 | * additionally, each type may have multiple memseg lists associated | |||||
2255 | * with it, each limited by either RTE_MAX_MEM_MB_PER_LIST for bigger | |||||
2256 | * page sizes, or RTE_MAX_MEMSEG_PER_LIST segments for smaller ones. | |||||
2257 | * | |||||
2258 | * the number of memseg lists per type is decided based on the above | |||||
2259 | * limits, and also taking number of detected NUMA nodes, to make sure | |||||
2260 | * that we don't run out of memseg lists before we populate all NUMA | |||||
2261 | * nodes with memory. | |||||
2262 | * | |||||
2263 | * we do this in three stages. first, we collect the number of types. | |||||
2264 | * then, we figure out memory constraints and populate the list of | |||||
2265 | * would-be memseg lists. then, we go ahead and allocate the memseg | |||||
2266 | * lists. | |||||
2267 | */ | |||||
2268 | ||||||
2269 | /* create space for mem types */ | |||||
2270 | n_memtypes = internal_config.num_hugepage_sizes * rte_socket_count(); | |||||
2271 | memtypes = calloc(n_memtypes, sizeof(*memtypes)); | |||||
2272 | if (memtypes == NULL((void*)0)) { | |||||
2273 | RTE_LOG(ERR, EAL, "Cannot allocate space for memory types\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate space for memory types\n" ); | |||||
2274 | return -1; | |||||
2275 | } | |||||
2276 | ||||||
2277 | /* populate mem types */ | |||||
2278 | cur_type = 0; | |||||
2279 | for (hpi_idx = 0; hpi_idx < (int) internal_config.num_hugepage_sizes; | |||||
2280 | hpi_idx++) { | |||||
2281 | struct hugepage_info *hpi; | |||||
2282 | uint64_t hugepage_sz; | |||||
2283 | ||||||
2284 | hpi = &internal_config.hugepage_info[hpi_idx]; | |||||
2285 | hugepage_sz = hpi->hugepage_sz; | |||||
2286 | ||||||
2287 | for (i = 0; i < (int) rte_socket_count(); i++, cur_type++) { | |||||
2288 | int socket_id = rte_socket_id_by_idx(i); | |||||
2289 | ||||||
2290 | #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
2291 | /* we can still sort pages by socket in legacy mode */ | |||||
2292 | if (!internal_config.legacy_mem && socket_id > 0) | |||||
2293 | break; | |||||
2294 | #endif | |||||
2295 | memtypes[cur_type].page_sz = hugepage_sz; | |||||
2296 | memtypes[cur_type].socket_id = socket_id; | |||||
2297 | ||||||
2298 | RTE_LOG(DEBUG, EAL, "Detected memory type: "rte_log(8U, 0, "EAL" ": " "Detected memory type: " "socket_id:%u hugepage_sz:%" "l" "u" "\n", socket_id, hugepage_sz) | |||||
2299 | "socket_id:%u hugepage_sz:%" PRIu64 "\n",rte_log(8U, 0, "EAL" ": " "Detected memory type: " "socket_id:%u hugepage_sz:%" "l" "u" "\n", socket_id, hugepage_sz) | |||||
2300 | socket_id, hugepage_sz)rte_log(8U, 0, "EAL" ": " "Detected memory type: " "socket_id:%u hugepage_sz:%" "l" "u" "\n", socket_id, hugepage_sz); | |||||
2301 | } | |||||
2302 | } | |||||
2303 | /* number of memtypes could have been lower due to no NUMA support */ | |||||
2304 | n_memtypes = cur_type; | |||||
2305 | ||||||
2306 | /* set up limits for types */ | |||||
2307 | max_mem = (uint64_t)RTE_MAX_MEM_MB524288 << 20; | |||||
2308 | max_mem_per_type = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20,__extension__ ({ __typeof__ ((uint64_t)131072 << 20) _a = ((uint64_t)131072 << 20); __typeof__ (max_mem / n_memtypes ) _b = (max_mem / n_memtypes); _a < _b ? _a : _b; }) | |||||
| ||||||
2309 | max_mem / n_memtypes)__extension__ ({ __typeof__ ((uint64_t)131072 << 20) _a = ((uint64_t)131072 << 20); __typeof__ (max_mem / n_memtypes ) _b = (max_mem / n_memtypes); _a < _b ? _a : _b; }); | |||||
2310 | /* | |||||
2311 | * limit maximum number of segment lists per type to ensure there's | |||||
2312 | * space for memseg lists for all NUMA nodes with all page sizes | |||||
2313 | */ | |||||
2314 | max_seglists_per_type = RTE_MAX_MEMSEG_LISTS64 / n_memtypes; | |||||
2315 | ||||||
2316 | if (max_seglists_per_type == 0) { | |||||
2317 | RTE_LOG(ERR, EAL, "Cannot accommodate all memory types, please increase %s\n",rte_log(4U, 0, "EAL" ": " "Cannot accommodate all memory types, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS") | |||||
2318 | RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "Cannot accommodate all memory types, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS"); | |||||
2319 | goto out; | |||||
2320 | } | |||||
2321 | ||||||
2322 | /* go through all mem types and create segment lists */ | |||||
2323 | msl_idx = 0; | |||||
2324 | for (cur_type = 0; cur_type < n_memtypes; cur_type++) { | |||||
2325 | unsigned int cur_seglist, n_seglists, n_segs; | |||||
2326 | unsigned int max_segs_per_type, max_segs_per_list; | |||||
2327 | struct memtype *type = &memtypes[cur_type]; | |||||
2328 | uint64_t max_mem_per_list, pagesz; | |||||
2329 | int socket_id; | |||||
2330 | ||||||
2331 | pagesz = type->page_sz; | |||||
2332 | socket_id = type->socket_id; | |||||
2333 | ||||||
2334 | /* | |||||
2335 | * we need to create segment lists for this type. we must take | |||||
2336 | * into account the following things: | |||||
2337 | * | |||||
2338 | * 1. total amount of memory we can use for this memory type | |||||
2339 | * 2. total amount of memory per memseg list allowed | |||||
2340 | * 3. number of segments needed to fit the amount of memory | |||||
2341 | * 4. number of segments allowed per type | |||||
2342 | * 5. number of segments allowed per memseg list | |||||
2343 | * 6. number of memseg lists we are allowed to take up | |||||
2344 | */ | |||||
2345 | ||||||
2346 | /* calculate how much segments we will need in total */ | |||||
2347 | max_segs_per_type = max_mem_per_type / pagesz; | |||||
2348 | /* limit number of segments to maximum allowed per type */ | |||||
2349 | max_segs_per_type = RTE_MIN(max_segs_per_type,__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type ); __typeof__ ((unsigned int)32768) _b = ((unsigned int)32768 ); _a < _b ? _a : _b; }) | |||||
2350 | (unsigned int)RTE_MAX_MEMSEG_PER_TYPE)__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type ); __typeof__ ((unsigned int)32768) _b = ((unsigned int)32768 ); _a < _b ? _a : _b; }); | |||||
2351 | /* limit number of segments to maximum allowed per list */ | |||||
2352 | max_segs_per_list = RTE_MIN(max_segs_per_type,__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type ); __typeof__ ((unsigned int)8192) _b = ((unsigned int)8192); _a < _b ? _a : _b; }) | |||||
2353 | (unsigned int)RTE_MAX_MEMSEG_PER_LIST)__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type ); __typeof__ ((unsigned int)8192) _b = ((unsigned int)8192); _a < _b ? _a : _b; }); | |||||
2354 | ||||||
2355 | /* calculate how much memory we can have per segment list */ | |||||
2356 | max_mem_per_list = RTE_MIN(max_segs_per_list * pagesz,__extension__ ({ __typeof__ (max_segs_per_list * pagesz) _a = (max_segs_per_list * pagesz); __typeof__ ((uint64_t)32768 << 20) _b = ((uint64_t)32768 << 20); _a < _b ? _a : _b ; }) | |||||
2357 | (uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20)__extension__ ({ __typeof__ (max_segs_per_list * pagesz) _a = (max_segs_per_list * pagesz); __typeof__ ((uint64_t)32768 << 20) _b = ((uint64_t)32768 << 20); _a < _b ? _a : _b ; }); | |||||
2358 | ||||||
2359 | /* calculate how many segments each segment list will have */ | |||||
2360 | n_segs = RTE_MIN(max_segs_per_list, max_mem_per_list / pagesz)__extension__ ({ __typeof__ (max_segs_per_list) _a = (max_segs_per_list ); __typeof__ (max_mem_per_list / pagesz) _b = (max_mem_per_list / pagesz); _a < _b ? _a : _b; }); | |||||
2361 | ||||||
2362 | /* calculate how many segment lists we can have */ | |||||
2363 | n_seglists = RTE_MIN(max_segs_per_type / n_segs,__extension__ ({ __typeof__ (max_segs_per_type / n_segs) _a = (max_segs_per_type / n_segs); __typeof__ (max_mem_per_type / max_mem_per_list) _b = (max_mem_per_type / max_mem_per_list) ; _a < _b ? _a : _b; }) | |||||
2364 | max_mem_per_type / max_mem_per_list)__extension__ ({ __typeof__ (max_segs_per_type / n_segs) _a = (max_segs_per_type / n_segs); __typeof__ (max_mem_per_type / max_mem_per_list) _b = (max_mem_per_type / max_mem_per_list) ; _a < _b ? _a : _b; }); | |||||
2365 | ||||||
2366 | /* limit number of segment lists according to our maximum */ | |||||
2367 | n_seglists = RTE_MIN(n_seglists, max_seglists_per_type)__extension__ ({ __typeof__ (n_seglists) _a = (n_seglists); __typeof__ (max_seglists_per_type) _b = (max_seglists_per_type); _a < _b ? _a : _b; }); | |||||
2368 | ||||||
2369 | RTE_LOG(DEBUG, EAL, "Creating %i segment lists: "rte_log(8U, 0, "EAL" ": " "Creating %i segment lists: " "n_segs:%i socket_id:%i hugepage_sz:%" "l" "u" "\n", n_seglists, n_segs, socket_id, pagesz) | |||||
2370 | "n_segs:%i socket_id:%i hugepage_sz:%" PRIu64 "\n",rte_log(8U, 0, "EAL" ": " "Creating %i segment lists: " "n_segs:%i socket_id:%i hugepage_sz:%" "l" "u" "\n", n_seglists, n_segs, socket_id, pagesz) | |||||
2371 | n_seglists, n_segs, socket_id, pagesz)rte_log(8U, 0, "EAL" ": " "Creating %i segment lists: " "n_segs:%i socket_id:%i hugepage_sz:%" "l" "u" "\n", n_seglists, n_segs, socket_id, pagesz); | |||||
2372 | ||||||
2373 | /* create all segment lists */ | |||||
2374 | for (cur_seglist = 0; cur_seglist < n_seglists; cur_seglist++) { | |||||
2375 | if (msl_idx >= RTE_MAX_MEMSEG_LISTS64) { | |||||
2376 | RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS") | |||||
2377 | "No more space in memseg lists, please increase %s\n",rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS") | |||||
2378 | RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n" , "CONFIG_RTE_MAX_MEMSEG_LISTS"); | |||||
2379 | goto out; | |||||
2380 | } | |||||
2381 | msl = &mcfg->memsegs[msl_idx++]; | |||||
2382 | ||||||
2383 | if (alloc_memseg_list(msl, pagesz, n_segs, | |||||
2384 | socket_id, cur_seglist)) | |||||
2385 | goto out; | |||||
2386 | ||||||
2387 | if (alloc_va_space(msl)) { | |||||
2388 | RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space for memseg list\n" ); | |||||
2389 | goto out; | |||||
2390 | } | |||||
2391 | } | |||||
2392 | } | |||||
2393 | /* we're successful */ | |||||
2394 | ret = 0; | |||||
2395 | out: | |||||
2396 | free(memtypes); | |||||
2397 | return ret; | |||||
2398 | } | |||||
2399 | ||||||
2400 | static int | |||||
2401 | memseg_secondary_init(void) | |||||
2402 | { | |||||
2403 | struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; | |||||
2404 | int msl_idx = 0; | |||||
2405 | struct rte_memseg_list *msl; | |||||
2406 | ||||||
2407 | for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS64; msl_idx++) { | |||||
2408 | ||||||
2409 | msl = &mcfg->memsegs[msl_idx]; | |||||
2410 | ||||||
2411 | /* skip empty memseg lists */ | |||||
2412 | if (msl->memseg_arr.len == 0) | |||||
2413 | continue; | |||||
2414 | ||||||
2415 | if (rte_fbarray_attach(&msl->memseg_arr)) { | |||||
2416 | RTE_LOG(ERR, EAL, "Cannot attach to primary process memseg lists\n")rte_log(4U, 0, "EAL" ": " "Cannot attach to primary process memseg lists\n" ); | |||||
2417 | return -1; | |||||
2418 | } | |||||
2419 | ||||||
2420 | /* preallocate VA space */ | |||||
2421 | if (alloc_va_space(msl)) { | |||||
2422 | RTE_LOG(ERR, EAL, "Cannot preallocate VA space for hugepage memory\n")rte_log(4U, 0, "EAL" ": " "Cannot preallocate VA space for hugepage memory\n" ); | |||||
2423 | return -1; | |||||
2424 | } | |||||
2425 | } | |||||
2426 | ||||||
2427 | return 0; | |||||
2428 | } | |||||
2429 | ||||||
2430 | int | |||||
2431 | rte_eal_memseg_init(void) | |||||
2432 | { | |||||
2433 | /* increase rlimit to maximum */ | |||||
2434 | struct rlimit lim; | |||||
2435 | ||||||
2436 | if (getrlimit(RLIMIT_NOFILERLIMIT_NOFILE, &lim) == 0) { | |||||
| ||||||
2437 | /* set limit to maximum */ | |||||
2438 | lim.rlim_cur = lim.rlim_max; | |||||
2439 | ||||||
2440 | if (setrlimit(RLIMIT_NOFILERLIMIT_NOFILE, &lim) < 0) { | |||||
2441 | RTE_LOG(DEBUG, EAL, "Setting maximum number of open files failed: %s\n",rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files failed: %s\n" , strerror((*__errno_location ()))) | |||||
2442 | strerror(errno))rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files failed: %s\n" , strerror((*__errno_location ()))); | |||||
2443 | } else { | |||||
2444 | RTE_LOG(DEBUG, EAL, "Setting maximum number of open files to %"rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files to %" "l" "u" "\n", (uint64_t)lim.rlim_cur) | |||||
2445 | PRIu64 "\n",rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files to %" "l" "u" "\n", (uint64_t)lim.rlim_cur) | |||||
2446 | (uint64_t)lim.rlim_cur)rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files to %" "l" "u" "\n", (uint64_t)lim.rlim_cur); | |||||
2447 | } | |||||
2448 | } else { | |||||
2449 | RTE_LOG(ERR, EAL, "Cannot get current resource limits\n")rte_log(4U, 0, "EAL" ": " "Cannot get current resource limits\n" ); | |||||
2450 | } | |||||
2451 | #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES | |||||
2452 | if (!internal_config.legacy_mem && rte_socket_count() > 1) { | |||||
2453 | RTE_LOG(WARNING, EAL, "DPDK is running on a NUMA system, but is compiled without NUMA support.\n")rte_log(5U, 0, "EAL" ": " "DPDK is running on a NUMA system, but is compiled without NUMA support.\n" ); | |||||
2454 | RTE_LOG(WARNING, EAL, "This will have adverse consequences for performance and usability.\n")rte_log(5U, 0, "EAL" ": " "This will have adverse consequences for performance and usability.\n" ); | |||||
2455 | RTE_LOG(WARNING, EAL, "Please use --"OPT_LEGACY_MEM" option, or recompile with NUMA support.\n")rte_log(5U, 0, "EAL" ": " "Please use --""legacy-mem"" option, or recompile with NUMA support.\n" ); | |||||
2456 | } | |||||
2457 | #endif | |||||
2458 | ||||||
2459 | return rte_eal_process_type() == RTE_PROC_PRIMARY ? | |||||
2460 | #ifndef RTE_ARCH_641 | |||||
2461 | memseg_primary_init_32() : | |||||
2462 | #else | |||||
2463 | memseg_primary_init() : | |||||
2464 | #endif | |||||
2465 | memseg_secondary_init(); | |||||
2466 | } |