Bug Summary

File:home/bhubbard/working/src/ceph/src/spdk/dpdk/lib/librte_eal/linux/eal/eal_memory.c
Warning:line 1977, column 2
Value stored to 'cur_seg' is never read

Annotated Source Code

[?] Use j/k keys for keyboard navigation

1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 * Copyright(c) 2013 6WIND S.A.
4 */
5
6#define _FILE_OFFSET_BITS64 64
7#include <errno(*__errno_location ()).h>
8#include <fcntl.h>
9#include <stdarg.h>
10#include <stdbool.h>
11#include <stdlib.h>
12#include <stdio.h>
13#include <stdint.h>
14#include <inttypes.h>
15#include <string.h>
16#include <sys/mman.h>
17#include <sys/types.h>
18#include <sys/stat.h>
19#include <sys/queue.h>
20#include <sys/file.h>
21#include <sys/resource.h>
22#include <unistd.h>
23#include <limits.h>
24#include <sys/ioctl.h>
25#include <sys/time.h>
26#include <signal.h>
27#include <setjmp.h>
28#ifdef F_ADD_SEALS /* if file sealing is supported, so is memfd */
29#include <linux1/memfd.h>
30#define MEMFD_SUPPORTED
31#endif
32#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
33#include <numa.h>
34#include <numaif.h>
35#endif
36
37#include <rte_errno(per_lcore__rte_errno).h>
38#include <rte_log.h>
39#include <rte_memory.h>
40#include <rte_launch.h>
41#include <rte_eal.h>
42#include <rte_eal_memconfig.h>
43#include <rte_per_lcore.h>
44#include <rte_lcore.h>
45#include <rte_common.h>
46#include <rte_string_fns.h>
47
48#include "eal_private.h"
49#include "eal_memalloc.h"
50#include "eal_internal_cfg.h"
51#include "eal_filesystem.h"
52#include "eal_hugepages.h"
53#include "eal_options.h"
54
55#define PFN_MASK_SIZE8 8
56
57/**
58 * @file
59 * Huge page mapping under linux
60 *
61 * To reserve a big contiguous amount of memory, we use the hugepage
62 * feature of linux. For that, we need to have hugetlbfs mounted. This
63 * code will create many files in this directory (one per page) and
64 * map them in virtual memory. For each page, we will retrieve its
65 * physical address and remap it in order to have a virtual contiguous
66 * zone as well as a physical contiguous zone.
67 */
68
69static bool_Bool phys_addrs_available = true1;
70
71#define RANDOMIZE_VA_SPACE_FILE"/proc/sys/kernel/randomize_va_space" "/proc/sys/kernel/randomize_va_space"
72
73static void
74test_phys_addrs_available(void)
75{
76 uint64_t tmp = 0;
77 phys_addr_t physaddr;
78
79 if (!rte_eal_has_hugepages()) {
80 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "Started without hugepages support, physical addresses not available\n"
)
81 "Started without hugepages support, physical addresses not available\n")rte_log(4U, 0, "EAL" ": " "Started without hugepages support, physical addresses not available\n"
);
82 phys_addrs_available = false0;
83 return;
84 }
85
86 physaddr = rte_mem_virt2phy(&tmp);
87 if (physaddr == RTE_BAD_PHYS_ADDR((phys_addr_t)-1)) {
88 if (rte_eal_iova_mode() == RTE_IOVA_PA)
89 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. "
"Only vfio will function.\n", strerror((*__errno_location ()
)))
90 "Cannot obtain physical addresses: %s. "rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. "
"Only vfio will function.\n", strerror((*__errno_location ()
)))
91 "Only vfio will function.\n",rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. "
"Only vfio will function.\n", strerror((*__errno_location ()
)))
92 strerror(errno))rte_log(4U, 0, "EAL" ": " "Cannot obtain physical addresses: %s. "
"Only vfio will function.\n", strerror((*__errno_location ()
)));
93 phys_addrs_available = false0;
94 }
95}
96
97/*
98 * Get physical address of any mapped virtual address in the current process.
99 */
100phys_addr_t
101rte_mem_virt2phy(const void *virtaddr)
102{
103 int fd, retval;
104 uint64_t page, physaddr;
105 unsigned long virt_pfn;
106 int page_size;
107 off_t offset;
108
109 /* Cannot parse /proc/self/pagemap, no need to log errors everywhere */
110 if (!phys_addrs_available)
111 return RTE_BAD_IOVA((rte_iova_t)-1);
112
113 /* standard page size */
114 page_size = getpagesize();
115
116 fd = open("/proc/self/pagemap", O_RDONLY00);
117 if (fd < 0) {
118 RTE_LOG(INFO, EAL, "%s(): cannot open /proc/self/pagemap: %s\n",rte_log(7U, 0, "EAL" ": " "%s(): cannot open /proc/self/pagemap: %s\n"
, __func__, strerror((*__errno_location ())))
119 __func__, strerror(errno))rte_log(7U, 0, "EAL" ": " "%s(): cannot open /proc/self/pagemap: %s\n"
, __func__, strerror((*__errno_location ())));
120 return RTE_BAD_IOVA((rte_iova_t)-1);
121 }
122
123 virt_pfn = (unsigned long)virtaddr / page_size;
124 offset = sizeof(uint64_t) * virt_pfn;
125 if (lseek(fd, offset, SEEK_SET0) == (off_t) -1) {
126 RTE_LOG(INFO, EAL, "%s(): seek error in /proc/self/pagemap: %s\n",rte_log(7U, 0, "EAL" ": " "%s(): seek error in /proc/self/pagemap: %s\n"
, __func__, strerror((*__errno_location ())))
127 __func__, strerror(errno))rte_log(7U, 0, "EAL" ": " "%s(): seek error in /proc/self/pagemap: %s\n"
, __func__, strerror((*__errno_location ())));
128 close(fd);
129 return RTE_BAD_IOVA((rte_iova_t)-1);
130 }
131
132 retval = read(fd, &page, PFN_MASK_SIZE8);
133 close(fd);
134 if (retval < 0) {
135 RTE_LOG(INFO, EAL, "%s(): cannot read /proc/self/pagemap: %s\n",rte_log(7U, 0, "EAL" ": " "%s(): cannot read /proc/self/pagemap: %s\n"
, __func__, strerror((*__errno_location ())))
136 __func__, strerror(errno))rte_log(7U, 0, "EAL" ": " "%s(): cannot read /proc/self/pagemap: %s\n"
, __func__, strerror((*__errno_location ())));
137 return RTE_BAD_IOVA((rte_iova_t)-1);
138 } else if (retval != PFN_MASK_SIZE8) {
139 RTE_LOG(INFO, EAL, "%s(): read %d bytes from /proc/self/pagemap "rte_log(7U, 0, "EAL" ": " "%s(): read %d bytes from /proc/self/pagemap "
"but expected %d:\n", __func__, retval, 8)
140 "but expected %d:\n",rte_log(7U, 0, "EAL" ": " "%s(): read %d bytes from /proc/self/pagemap "
"but expected %d:\n", __func__, retval, 8)
141 __func__, retval, PFN_MASK_SIZE)rte_log(7U, 0, "EAL" ": " "%s(): read %d bytes from /proc/self/pagemap "
"but expected %d:\n", __func__, retval, 8);
142 return RTE_BAD_IOVA((rte_iova_t)-1);
143 }
144
145 /*
146 * the pfn (page frame number) are bits 0-54 (see
147 * pagemap.txt in linux Documentation)
148 */
149 if ((page & 0x7fffffffffffffULL) == 0)
150 return RTE_BAD_IOVA((rte_iova_t)-1);
151
152 physaddr = ((page & 0x7fffffffffffffULL) * page_size)
153 + ((unsigned long)virtaddr % page_size);
154
155 return physaddr;
156}
157
158rte_iova_t
159rte_mem_virt2iova(const void *virtaddr)
160{
161 if (rte_eal_iova_mode() == RTE_IOVA_VA)
162 return (uintptr_t)virtaddr;
163 return rte_mem_virt2phy(virtaddr);
164}
165
166/*
167 * For each hugepage in hugepg_tbl, fill the physaddr value. We find
168 * it by browsing the /proc/self/pagemap special file.
169 */
170static int
171find_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
172{
173 unsigned int i;
174 phys_addr_t addr;
175
176 for (i = 0; i < hpi->num_pages[0]; i++) {
177 addr = rte_mem_virt2phy(hugepg_tbl[i].orig_va);
178 if (addr == RTE_BAD_PHYS_ADDR((phys_addr_t)-1))
179 return -1;
180 hugepg_tbl[i].physaddr = addr;
181 }
182 return 0;
183}
184
185/*
186 * For each hugepage in hugepg_tbl, fill the physaddr value sequentially.
187 */
188static int
189set_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
190{
191 unsigned int i;
192 static phys_addr_t addr;
193
194 for (i = 0; i < hpi->num_pages[0]; i++) {
195 hugepg_tbl[i].physaddr = addr;
196 addr += hugepg_tbl[i].size;
197 }
198 return 0;
199}
200
201/*
202 * Check whether address-space layout randomization is enabled in
203 * the kernel. This is important for multi-process as it can prevent
204 * two processes mapping data to the same virtual address
205 * Returns:
206 * 0 - address space randomization disabled
207 * 1/2 - address space randomization enabled
208 * negative error code on error
209 */
210static int
211aslr_enabled(void)
212{
213 char c;
214 int retval, fd = open(RANDOMIZE_VA_SPACE_FILE"/proc/sys/kernel/randomize_va_space", O_RDONLY00);
215 if (fd < 0)
216 return -errno(*__errno_location ());
217 retval = read(fd, &c, 1);
218 close(fd);
219 if (retval < 0)
220 return -errno(*__errno_location ());
221 if (retval == 0)
222 return -EIO5;
223 switch (c) {
224 case '0' : return 0;
225 case '1' : return 1;
226 case '2' : return 2;
227 default: return -EINVAL22;
228 }
229}
230
231static sigjmp_buf huge_jmpenv;
232
233static void huge_sigbus_handler(int signo __rte_unused__attribute__((__unused__)))
234{
235 siglongjmp(huge_jmpenv, 1);
236}
237
238/* Put setjmp into a wrap method to avoid compiling error. Any non-volatile,
239 * non-static local variable in the stack frame calling sigsetjmp might be
240 * clobbered by a call to longjmp.
241 */
242static int huge_wrap_sigsetjmp(void)
243{
244 return sigsetjmp(huge_jmpenv, 1)__sigsetjmp (huge_jmpenv, 1);
245}
246
247#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
248/* Callback for numa library. */
249void numa_error(char *where)
250{
251 RTE_LOG(ERR, EAL, "%s failed: %s\n", where, strerror(errno))rte_log(4U, 0, "EAL" ": " "%s failed: %s\n", where, strerror(
(*__errno_location ())));
252}
253#endif
254
255/*
256 * Mmap all hugepages of hugepage table: it first open a file in
257 * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the
258 * virtual address is stored in hugepg_tbl[i].orig_va, else it is stored
259 * in hugepg_tbl[i].final_va. The second mapping (when orig is 0) tries to
260 * map contiguous physical blocks in contiguous virtual blocks.
261 */
262static unsigned
263map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi,
264 uint64_t *essential_memory __rte_unused__attribute__((__unused__)))
265{
266 int fd;
267 unsigned i;
268 void *virtaddr;
269#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
270 int node_id = -1;
271 int essential_prev = 0;
272 int oldpolicy;
273 struct bitmask *oldmask = NULL((void*)0);
274 bool_Bool have_numa = true1;
275 unsigned long maxnode = 0;
276
277 /* Check if kernel supports NUMA. */
278 if (numa_available() != 0) {
279 RTE_LOG(DEBUG, EAL, "NUMA is not supported.\n")rte_log(8U, 0, "EAL" ": " "NUMA is not supported.\n");
280 have_numa = false0;
281 }
282
283 if (have_numa) {
284 RTE_LOG(DEBUG, EAL, "Trying to obtain current memory policy.\n")rte_log(8U, 0, "EAL" ": " "Trying to obtain current memory policy.\n"
);
285 oldmask = numa_allocate_nodemask();
286 if (get_mempolicy(&oldpolicy, oldmask->maskp,
287 oldmask->size + 1, 0, 0) < 0) {
288 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "Failed to get current mempolicy: %s. "
"Assuming MPOL_DEFAULT.\n", strerror((*__errno_location ()))
)
289 "Failed to get current mempolicy: %s. "rte_log(4U, 0, "EAL" ": " "Failed to get current mempolicy: %s. "
"Assuming MPOL_DEFAULT.\n", strerror((*__errno_location ()))
)
290 "Assuming MPOL_DEFAULT.\n", strerror(errno))rte_log(4U, 0, "EAL" ": " "Failed to get current mempolicy: %s. "
"Assuming MPOL_DEFAULT.\n", strerror((*__errno_location ()))
);
291 oldpolicy = MPOL_DEFAULT;
292 }
293 for (i = 0; i < RTE_MAX_NUMA_NODES8; i++)
294 if (internal_config.socket_mem[i])
295 maxnode = i + 1;
296 }
297#endif
298
299 for (i = 0; i < hpi->num_pages[0]; i++) {
300 struct hugepage_file *hf = &hugepg_tbl[i];
301 uint64_t hugepage_sz = hpi->hugepage_sz;
302
303#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
304 if (maxnode) {
305 unsigned int j;
306
307 for (j = 0; j < maxnode; j++)
308 if (essential_memory[j])
309 break;
310
311 if (j == maxnode) {
312 node_id = (node_id + 1) % maxnode;
313 while (!internal_config.socket_mem[node_id]) {
314 node_id++;
315 node_id %= maxnode;
316 }
317 essential_prev = 0;
318 } else {
319 node_id = j;
320 essential_prev = essential_memory[j];
321
322 if (essential_memory[j] < hugepage_sz)
323 essential_memory[j] = 0;
324 else
325 essential_memory[j] -= hugepage_sz;
326 }
327
328 RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Setting policy MPOL_PREFERRED for socket %d\n"
, node_id)
329 "Setting policy MPOL_PREFERRED for socket %d\n",rte_log(8U, 0, "EAL" ": " "Setting policy MPOL_PREFERRED for socket %d\n"
, node_id)
330 node_id)rte_log(8U, 0, "EAL" ": " "Setting policy MPOL_PREFERRED for socket %d\n"
, node_id);
331 numa_set_preferred(node_id);
332 }
333#endif
334
335 hf->file_id = i;
336 hf->size = hugepage_sz;
337 eal_get_hugefile_path(hf->filepath, sizeof(hf->filepath),
338 hpi->hugedir, hf->file_id);
339 hf->filepath[sizeof(hf->filepath) - 1] = '\0';
340
341 /* try to create hugepage file */
342 fd = open(hf->filepath, O_CREAT0100 | O_RDWR02, 0600);
343 if (fd < 0) {
344 RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__,rte_log(8U, 0, "EAL" ": " "%s(): open failed: %s\n", __func__
, strerror((*__errno_location ())))
345 strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): open failed: %s\n", __func__
, strerror((*__errno_location ())));
346 goto out;
347 }
348
349 /* map the segment, and populate page tables,
350 * the kernel fills this segment with zeros. we don't care where
351 * this gets mapped - we already have contiguous memory areas
352 * ready for us to map into.
353 */
354 virtaddr = mmap(NULL((void*)0), hugepage_sz, PROT_READ0x1 | PROT_WRITE0x2,
355 MAP_SHARED0x01 | MAP_POPULATE0x08000, fd, 0);
356 if (virtaddr == MAP_FAILED((void *) -1)) {
357 RTE_LOG(DEBUG, EAL, "%s(): mmap failed: %s\n", __func__,rte_log(8U, 0, "EAL" ": " "%s(): mmap failed: %s\n", __func__
, strerror((*__errno_location ())))
358 strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): mmap failed: %s\n", __func__
, strerror((*__errno_location ())));
359 close(fd);
360 goto out;
361 }
362
363 hf->orig_va = virtaddr;
364
365 /* In linux, hugetlb limitations, like cgroup, are
366 * enforced at fault time instead of mmap(), even
367 * with the option of MAP_POPULATE. Kernel will send
368 * a SIGBUS signal. To avoid to be killed, save stack
369 * environment here, if SIGBUS happens, we can jump
370 * back here.
371 */
372 if (huge_wrap_sigsetjmp()) {
373 RTE_LOG(DEBUG, EAL, "SIGBUS: Cannot mmap more "rte_log(8U, 0, "EAL" ": " "SIGBUS: Cannot mmap more " "hugepages of size %u MB\n"
, (unsigned int)(hugepage_sz / 0x100000))
374 "hugepages of size %u MB\n",rte_log(8U, 0, "EAL" ": " "SIGBUS: Cannot mmap more " "hugepages of size %u MB\n"
, (unsigned int)(hugepage_sz / 0x100000))
375 (unsigned int)(hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "SIGBUS: Cannot mmap more " "hugepages of size %u MB\n"
, (unsigned int)(hugepage_sz / 0x100000));
376 munmap(virtaddr, hugepage_sz);
377 close(fd);
378 unlink(hugepg_tbl[i].filepath);
379#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
380 if (maxnode)
381 essential_memory[node_id] =
382 essential_prev;
383#endif
384 goto out;
385 }
386 *(int *)virtaddr = 0;
387
388 /* set shared lock on the file. */
389 if (flock(fd, LOCK_SH1) < 0) {
390 RTE_LOG(DEBUG, EAL, "%s(): Locking file failed:%s \n",rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed:%s \n", __func__
, strerror((*__errno_location ())))
391 __func__, strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed:%s \n", __func__
, strerror((*__errno_location ())));
392 close(fd);
393 goto out;
394 }
395
396 close(fd);
397 }
398
399out:
400#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
401 if (maxnode) {
402 RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Restoring previous memory policy: %d\n"
, oldpolicy)
403 "Restoring previous memory policy: %d\n", oldpolicy)rte_log(8U, 0, "EAL" ": " "Restoring previous memory policy: %d\n"
, oldpolicy);
404 if (oldpolicy == MPOL_DEFAULT) {
405 numa_set_localalloc();
406 } else if (set_mempolicy(oldpolicy, oldmask->maskp,
407 oldmask->size + 1) < 0) {
408 RTE_LOG(ERR, EAL, "Failed to restore mempolicy: %s\n",rte_log(4U, 0, "EAL" ": " "Failed to restore mempolicy: %s\n"
, strerror((*__errno_location ())))
409 strerror(errno))rte_log(4U, 0, "EAL" ": " "Failed to restore mempolicy: %s\n"
, strerror((*__errno_location ())));
410 numa_set_localalloc();
411 }
412 }
413 if (oldmask != NULL((void*)0))
414 numa_free_cpumask(oldmask);
415#endif
416 return i;
417}
418
419/*
420 * Parse /proc/self/numa_maps to get the NUMA socket ID for each huge
421 * page.
422 */
423static int
424find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
425{
426 int socket_id;
427 char *end, *nodestr;
428 unsigned i, hp_count = 0;
429 uint64_t virt_addr;
430 char buf[BUFSIZ8192];
431 char hugedir_str[PATH_MAX4096];
432 FILE *f;
433
434 f = fopen("/proc/self/numa_maps", "r");
435 if (f == NULL((void*)0)) {
436 RTE_LOG(NOTICE, EAL, "NUMA support not available"rte_log(6U, 0, "EAL" ": " "NUMA support not available" " consider that all memory is in socket_id 0\n"
)
437 " consider that all memory is in socket_id 0\n")rte_log(6U, 0, "EAL" ": " "NUMA support not available" " consider that all memory is in socket_id 0\n"
);
438 return 0;
439 }
440
441 snprintf(hugedir_str, sizeof(hugedir_str),
442 "%s/%s", hpi->hugedir, eal_get_hugefile_prefix());
443
444 /* parse numa map */
445 while (fgets(buf, sizeof(buf), f) != NULL((void*)0)) {
446
447 /* ignore non huge page */
448 if (strstr(buf, " huge ") == NULL((void*)0) &&
449 strstr(buf, hugedir_str) == NULL((void*)0))
450 continue;
451
452 /* get zone addr */
453 virt_addr = strtoull(buf, &end, 16);
454 if (virt_addr == 0 || end == buf) {
455 RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n"
, __func__);
456 goto error;
457 }
458
459 /* get node id (socket id) */
460 nodestr = strstr(buf, " N");
461 if (nodestr == NULL((void*)0)) {
462 RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n"
, __func__);
463 goto error;
464 }
465 nodestr += 2;
466 end = strstr(nodestr, "=");
467 if (end == NULL((void*)0)) {
468 RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n"
, __func__);
469 goto error;
470 }
471 end[0] = '\0';
472 end = NULL((void*)0);
473
474 socket_id = strtoul(nodestr, &end, 0);
475 if ((nodestr[0] == '\0') || (end == NULL((void*)0)) || (*end != '\0')) {
476 RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__)rte_log(4U, 0, "EAL" ": " "%s(): error in numa_maps parsing\n"
, __func__);
477 goto error;
478 }
479
480 /* if we find this page in our mappings, set socket_id */
481 for (i = 0; i < hpi->num_pages[0]; i++) {
482 void *va = (void *)(unsigned long)virt_addr;
483 if (hugepg_tbl[i].orig_va == va) {
484 hugepg_tbl[i].socket_id = socket_id;
485 hp_count++;
486#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
487 RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Hugepage %s is on socket %d\n", hugepg_tbl
[i].filepath, socket_id)
488 "Hugepage %s is on socket %d\n",rte_log(8U, 0, "EAL" ": " "Hugepage %s is on socket %d\n", hugepg_tbl
[i].filepath, socket_id)
489 hugepg_tbl[i].filepath, socket_id)rte_log(8U, 0, "EAL" ": " "Hugepage %s is on socket %d\n", hugepg_tbl
[i].filepath, socket_id);
490#endif
491 }
492 }
493 }
494
495 if (hp_count < hpi->num_pages[0])
496 goto error;
497
498 fclose(f);
499 return 0;
500
501error:
502 fclose(f);
503 return -1;
504}
505
506static int
507cmp_physaddr(const void *a, const void *b)
508{
509#ifndef RTE_ARCH_PPC_64
510 const struct hugepage_file *p1 = a;
511 const struct hugepage_file *p2 = b;
512#else
513 /* PowerPC needs memory sorted in reverse order from x86 */
514 const struct hugepage_file *p1 = b;
515 const struct hugepage_file *p2 = a;
516#endif
517 if (p1->physaddr < p2->physaddr)
518 return -1;
519 else if (p1->physaddr > p2->physaddr)
520 return 1;
521 else
522 return 0;
523}
524
525/*
526 * Uses mmap to create a shared memory area for storage of data
527 * Used in this file to store the hugepage file map on disk
528 */
529static void *
530create_shared_memory(const char *filename, const size_t mem_size)
531{
532 void *retval;
533 int fd;
534
535 /* if no shared files mode is used, create anonymous memory instead */
536 if (internal_config.no_shconf) {
537 retval = mmap(NULL((void*)0), mem_size, PROT_READ0x1 | PROT_WRITE0x2,
538 MAP_PRIVATE0x02 | MAP_ANONYMOUS0x20, -1, 0);
539 if (retval == MAP_FAILED((void *) -1))
540 return NULL((void*)0);
541 return retval;
542 }
543
544 fd = open(filename, O_CREAT0100 | O_RDWR02, 0600);
545 if (fd < 0)
546 return NULL((void*)0);
547 if (ftruncate(fd, mem_size) < 0) {
548 close(fd);
549 return NULL((void*)0);
550 }
551 retval = mmap(NULL((void*)0), mem_size, PROT_READ0x1 | PROT_WRITE0x2, MAP_SHARED0x01, fd, 0);
552 close(fd);
553 if (retval == MAP_FAILED((void *) -1))
554 return NULL((void*)0);
555 return retval;
556}
557
558/*
559 * this copies *active* hugepages from one hugepage table to another.
560 * destination is typically the shared memory.
561 */
562static int
563copy_hugepages_to_shared_mem(struct hugepage_file * dst, int dest_size,
564 const struct hugepage_file * src, int src_size)
565{
566 int src_pos, dst_pos = 0;
567
568 for (src_pos = 0; src_pos < src_size; src_pos++) {
569 if (src[src_pos].orig_va != NULL((void*)0)) {
570 /* error on overflow attempt */
571 if (dst_pos == dest_size)
572 return -1;
573 memcpy(&dst[dst_pos], &src[src_pos], sizeof(struct hugepage_file));
574 dst_pos++;
575 }
576 }
577 return 0;
578}
579
580static int
581unlink_hugepage_files(struct hugepage_file *hugepg_tbl,
582 unsigned num_hp_info)
583{
584 unsigned socket, size;
585 int page, nrpages = 0;
586
587 /* get total number of hugepages */
588 for (size = 0; size < num_hp_info; size++)
589 for (socket = 0; socket < RTE_MAX_NUMA_NODES8; socket++)
590 nrpages +=
591 internal_config.hugepage_info[size].num_pages[socket];
592
593 for (page = 0; page < nrpages; page++) {
594 struct hugepage_file *hp = &hugepg_tbl[page];
595
596 if (hp->orig_va != NULL((void*)0) && unlink(hp->filepath)) {
597 RTE_LOG(WARNING, EAL, "%s(): Removing %s failed: %s\n",rte_log(5U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__
, hp->filepath, strerror((*__errno_location ())))
598 __func__, hp->filepath, strerror(errno))rte_log(5U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__
, hp->filepath, strerror((*__errno_location ())));
599 }
600 }
601 return 0;
602}
603
604/*
605 * unmaps hugepages that are not going to be used. since we originally allocate
606 * ALL hugepages (not just those we need), additional unmapping needs to be done.
607 */
608static int
609unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl,
610 struct hugepage_info *hpi,
611 unsigned num_hp_info)
612{
613 unsigned socket, size;
614 int page, nrpages = 0;
615
616 /* get total number of hugepages */
617 for (size = 0; size < num_hp_info; size++)
618 for (socket = 0; socket < RTE_MAX_NUMA_NODES8; socket++)
619 nrpages += internal_config.hugepage_info[size].num_pages[socket];
620
621 for (size = 0; size < num_hp_info; size++) {
622 for (socket = 0; socket < RTE_MAX_NUMA_NODES8; socket++) {
623 unsigned pages_found = 0;
624
625 /* traverse until we have unmapped all the unused pages */
626 for (page = 0; page < nrpages; page++) {
627 struct hugepage_file *hp = &hugepg_tbl[page];
628
629 /* find a page that matches the criteria */
630 if ((hp->size == hpi[size].hugepage_sz) &&
631 (hp->socket_id == (int) socket)) {
632
633 /* if we skipped enough pages, unmap the rest */
634 if (pages_found == hpi[size].num_pages[socket]) {
635 uint64_t unmap_len;
636
637 unmap_len = hp->size;
638
639 /* get start addr and len of the remaining segment */
640 munmap(hp->orig_va,
641 (size_t)unmap_len);
642
643 hp->orig_va = NULL((void*)0);
644 if (unlink(hp->filepath) == -1) {
645 RTE_LOG(ERR, EAL, "%s(): Removing %s failed: %s\n",rte_log(4U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__
, hp->filepath, strerror((*__errno_location ())))
646 __func__, hp->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "%s(): Removing %s failed: %s\n", __func__
, hp->filepath, strerror((*__errno_location ())));
647 return -1;
648 }
649 } else {
650 /* lock the page and skip */
651 pages_found++;
652 }
653
654 } /* match page */
655 } /* foreach page */
656 } /* foreach socket */
657 } /* foreach pagesize */
658
659 return 0;
660}
661
662static int
663remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end)
664{
665 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
666 struct rte_memseg_list *msl;
667 struct rte_fbarray *arr;
668 int cur_page, seg_len;
669 unsigned int msl_idx;
670 int ms_idx;
671 uint64_t page_sz;
672 size_t memseg_len;
673 int socket_id;
674
675 page_sz = hugepages[seg_start].size;
676 socket_id = hugepages[seg_start].socket_id;
677 seg_len = seg_end - seg_start;
678
679 RTE_LOG(DEBUG, EAL, "Attempting to map %" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Attempting to map %" "l" "u" "M on socket %i\n"
, (seg_len * page_sz) >> 20ULL, socket_id)
680 (seg_len * page_sz) >> 20ULL, socket_id)rte_log(8U, 0, "EAL" ": " "Attempting to map %" "l" "u" "M on socket %i\n"
, (seg_len * page_sz) >> 20ULL, socket_id);
681
682 /* find free space in memseg lists */
683 for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS64; msl_idx++) {
684 bool_Bool empty;
685 msl = &mcfg->memsegs[msl_idx];
686 arr = &msl->memseg_arr;
687
688 if (msl->page_sz != page_sz)
689 continue;
690 if (msl->socket_id != socket_id)
691 continue;
692
693 /* leave space for a hole if array is not empty */
694 empty = arr->count == 0;
695 ms_idx = rte_fbarray_find_next_n_free(arr, 0,
696 seg_len + (empty ? 0 : 1));
697
698 /* memseg list is full? */
699 if (ms_idx < 0)
700 continue;
701
702 /* leave some space between memsegs, they are not IOVA
703 * contiguous, so they shouldn't be VA contiguous either.
704 */
705 if (!empty)
706 ms_idx++;
707 break;
708 }
709 if (msl_idx == RTE_MAX_MEMSEG_LISTS64) {
710 RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase %s and/or %s in configuration.\n",rte_log(4U, 0, "EAL" ": " "Could not find space for memseg. Please increase %s and/or %s in configuration.\n"
, "CONFIG_RTE_MAX_MEMSEG_PER_TYPE", "CONFIG_RTE_MAX_MEM_PER_TYPE"
)
711 RTE_STR(CONFIG_RTE_MAX_MEMSEG_PER_TYPE),rte_log(4U, 0, "EAL" ": " "Could not find space for memseg. Please increase %s and/or %s in configuration.\n"
, "CONFIG_RTE_MAX_MEMSEG_PER_TYPE", "CONFIG_RTE_MAX_MEM_PER_TYPE"
)
712 RTE_STR(CONFIG_RTE_MAX_MEM_PER_TYPE))rte_log(4U, 0, "EAL" ": " "Could not find space for memseg. Please increase %s and/or %s in configuration.\n"
, "CONFIG_RTE_MAX_MEMSEG_PER_TYPE", "CONFIG_RTE_MAX_MEM_PER_TYPE"
);
713 return -1;
714 }
715
716#ifdef RTE_ARCH_PPC64
717 /* for PPC64 we go through the list backwards */
718 for (cur_page = seg_end - 1; cur_page >= seg_start;
719 cur_page--, ms_idx++) {
720#else
721 for (cur_page = seg_start; cur_page < seg_end; cur_page++, ms_idx++) {
722#endif
723 struct hugepage_file *hfile = &hugepages[cur_page];
724 struct rte_memseg *ms = rte_fbarray_get(arr, ms_idx);
725 void *addr;
726 int fd;
727
728 fd = open(hfile->filepath, O_RDWR02);
729 if (fd < 0) {
730 RTE_LOG(ERR, EAL, "Could not open '%s': %s\n",rte_log(4U, 0, "EAL" ": " "Could not open '%s': %s\n", hfile->
filepath, strerror((*__errno_location ())))
731 hfile->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Could not open '%s': %s\n", hfile->
filepath, strerror((*__errno_location ())));
732 return -1;
733 }
734 /* set shared lock on the file. */
735 if (flock(fd, LOCK_SH1) < 0) {
736 RTE_LOG(DEBUG, EAL, "Could not lock '%s': %s\n",rte_log(8U, 0, "EAL" ": " "Could not lock '%s': %s\n", hfile->
filepath, strerror((*__errno_location ())))
737 hfile->filepath, strerror(errno))rte_log(8U, 0, "EAL" ": " "Could not lock '%s': %s\n", hfile->
filepath, strerror((*__errno_location ())));
738 close(fd);
739 return -1;
740 }
741 memseg_len = (size_t)page_sz;
742 addr = RTE_PTR_ADD(msl->base_va, ms_idx * memseg_len)((void*)((uintptr_t)(msl->base_va) + (ms_idx * memseg_len)
));
743
744 /* we know this address is already mmapped by memseg list, so
745 * using MAP_FIXED here is safe
746 */
747 addr = mmap(addr, page_sz, PROT_READ0x1 | PROT_WRITE0x2,
748 MAP_SHARED0x01 | MAP_POPULATE0x08000 | MAP_FIXED0x10, fd, 0);
749 if (addr == MAP_FAILED((void *) -1)) {
750 RTE_LOG(ERR, EAL, "Couldn't remap '%s': %s\n",rte_log(4U, 0, "EAL" ": " "Couldn't remap '%s': %s\n", hfile->
filepath, strerror((*__errno_location ())))
751 hfile->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Couldn't remap '%s': %s\n", hfile->
filepath, strerror((*__errno_location ())));
752 close(fd);
753 return -1;
754 }
755
756 /* we have a new address, so unmap previous one */
757#ifndef RTE_ARCH_641
758 /* in 32-bit legacy mode, we have already unmapped the page */
759 if (!internal_config.legacy_mem)
760 munmap(hfile->orig_va, page_sz);
761#else
762 munmap(hfile->orig_va, page_sz);
763#endif
764
765 hfile->orig_va = NULL((void*)0);
766 hfile->final_va = addr;
767
768 /* rewrite physical addresses in IOVA as VA mode */
769 if (rte_eal_iova_mode() == RTE_IOVA_VA)
770 hfile->physaddr = (uintptr_t)addr;
771
772 /* set up memseg data */
773 ms->addr = addr;
774 ms->hugepage_sz = page_sz;
775 ms->len = memseg_len;
776 ms->iova = hfile->physaddr;
777 ms->socket_id = hfile->socket_id;
778 ms->nchannel = rte_memory_get_nchannel();
779 ms->nrank = rte_memory_get_nrank();
780
781 rte_fbarray_set_used(arr, ms_idx);
782
783 /* store segment fd internally */
784 if (eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd) < 0)
785 RTE_LOG(ERR, EAL, "Could not store segment fd: %s\n",rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n",
rte_strerror((per_lcore__rte_errno)))
786 rte_strerror(rte_errno))rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n",
rte_strerror((per_lcore__rte_errno)));
787 }
788 RTE_LOG(DEBUG, EAL, "Allocated %" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Allocated %" "l" "u" "M on socket %i\n"
, (seg_len * page_sz) >> 20, socket_id)
789 (seg_len * page_sz) >> 20, socket_id)rte_log(8U, 0, "EAL" ": " "Allocated %" "l" "u" "M on socket %i\n"
, (seg_len * page_sz) >> 20, socket_id);
790 return 0;
791}
792
793static uint64_t
794get_mem_amount(uint64_t page_sz, uint64_t max_mem)
795{
796 uint64_t area_sz, max_pages;
797
798 /* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */
799 max_pages = RTE_MAX_MEMSEG_PER_LIST8192;
800 max_mem = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20, max_mem)__extension__ ({ __typeof__ ((uint64_t)32768 << 20) _a =
((uint64_t)32768 << 20); __typeof__ (max_mem) _b = (max_mem
); _a < _b ? _a : _b; });
801
802 area_sz = RTE_MIN(page_sz * max_pages, max_mem)__extension__ ({ __typeof__ (page_sz * max_pages) _a = (page_sz
* max_pages); __typeof__ (max_mem) _b = (max_mem); _a < _b
? _a : _b; });
803
804 /* make sure the list isn't smaller than the page size */
805 area_sz = RTE_MAX(area_sz, page_sz)__extension__ ({ __typeof__ (area_sz) _a = (area_sz); __typeof__
(page_sz) _b = (page_sz); _a > _b ? _a : _b; });
806
807 return RTE_ALIGN(area_sz, page_sz)(__typeof__(((area_sz) + ((__typeof__(area_sz)) (page_sz) - 1
))))((((area_sz) + ((__typeof__(area_sz)) (page_sz) - 1))) &
(~((__typeof__(((area_sz) + ((__typeof__(area_sz)) (page_sz)
- 1))))((page_sz) - 1))));
808}
809
810static int
811free_memseg_list(struct rte_memseg_list *msl)
812{
813 if (rte_fbarray_destroy(&msl->memseg_arr)) {
814 RTE_LOG(ERR, EAL, "Cannot destroy memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot destroy memseg list\n");
815 return -1;
816 }
817 memset(msl, 0, sizeof(*msl));
818 return 0;
819}
820
821#define MEMSEG_LIST_FMT"memseg-%" "l" "u" "k-%i-%i" "memseg-%" PRIu64"l" "u" "k-%i-%i"
822static int
823alloc_memseg_list(struct rte_memseg_list *msl, uint64_t page_sz,
824 int n_segs, int socket_id, int type_msl_idx)
825{
826 char name[RTE_FBARRAY_NAME_LEN64];
827
828 snprintf(name, sizeof(name), MEMSEG_LIST_FMT"memseg-%" "l" "u" "k-%i-%i", page_sz >> 10, socket_id,
829 type_msl_idx);
830 if (rte_fbarray_init(&msl->memseg_arr, name, n_segs,
831 sizeof(struct rte_memseg))) {
832 RTE_LOG(ERR, EAL, "Cannot allocate memseg list: %s\n",rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list: %s\n"
, rte_strerror((per_lcore__rte_errno)))
833 rte_strerror(rte_errno))rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list: %s\n"
, rte_strerror((per_lcore__rte_errno)));
834 return -1;
835 }
836
837 msl->page_sz = page_sz;
838 msl->socket_id = socket_id;
839 msl->base_va = NULL((void*)0);
840
841 RTE_LOG(DEBUG, EAL, "Memseg list allocated: 0x%zxkB at socket %i\n",rte_log(8U, 0, "EAL" ": " "Memseg list allocated: 0x%zxkB at socket %i\n"
, (size_t)page_sz >> 10, socket_id)
842 (size_t)page_sz >> 10, socket_id)rte_log(8U, 0, "EAL" ": " "Memseg list allocated: 0x%zxkB at socket %i\n"
, (size_t)page_sz >> 10, socket_id);
843
844 return 0;
845}
846
847static int
848alloc_va_space(struct rte_memseg_list *msl)
849{
850 uint64_t page_sz;
851 size_t mem_sz;
852 void *addr;
853 int flags = 0;
854
855 page_sz = msl->page_sz;
856 mem_sz = page_sz * msl->memseg_arr.len;
857
858 addr = eal_get_virtual_area(msl->base_va, &mem_sz, page_sz, 0, flags);
859 if (addr == NULL((void*)0)) {
860 if (rte_errno(per_lcore__rte_errno) == EADDRNOTAVAIL99)
861 RTE_LOG(ERR, EAL, "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n",rte_log(4U, 0, "EAL" ": " "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n"
, (unsigned long long)mem_sz, msl->base_va)
862 (unsigned long long)mem_sz, msl->base_va)rte_log(4U, 0, "EAL" ": " "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n"
, (unsigned long long)mem_sz, msl->base_va);
863 else
864 RTE_LOG(ERR, EAL, "Cannot reserve memory\n")rte_log(4U, 0, "EAL" ": " "Cannot reserve memory\n");
865 return -1;
866 }
867 msl->base_va = addr;
868 msl->len = mem_sz;
869
870 return 0;
871}
872
873/*
874 * Our VA space is not preallocated yet, so preallocate it here. We need to know
875 * how many segments there are in order to map all pages into one address space,
876 * and leave appropriate holes between segments so that rte_malloc does not
877 * concatenate them into one big segment.
878 *
879 * we also need to unmap original pages to free up address space.
880 */
881static int __rte_unused__attribute__((__unused__))
882prealloc_segments(struct hugepage_file *hugepages, int n_pages)
883{
884 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
885 int cur_page, seg_start_page, end_seg, new_memseg;
886 unsigned int hpi_idx, socket, i;
887 int n_contig_segs, n_segs;
888 int msl_idx;
889
890 /* before we preallocate segments, we need to free up our VA space.
891 * we're not removing files, and we already have information about
892 * PA-contiguousness, so it is safe to unmap everything.
893 */
894 for (cur_page = 0; cur_page < n_pages; cur_page++) {
895 struct hugepage_file *hpi = &hugepages[cur_page];
896 munmap(hpi->orig_va, hpi->size);
897 hpi->orig_va = NULL((void*)0);
898 }
899
900 /* we cannot know how many page sizes and sockets we have discovered, so
901 * loop over all of them
902 */
903 for (hpi_idx = 0; hpi_idx < internal_config.num_hugepage_sizes;
904 hpi_idx++) {
905 uint64_t page_sz =
906 internal_config.hugepage_info[hpi_idx].hugepage_sz;
907
908 for (i = 0; i < rte_socket_count(); i++) {
909 struct rte_memseg_list *msl;
910
911 socket = rte_socket_id_by_idx(i);
912 n_contig_segs = 0;
913 n_segs = 0;
914 seg_start_page = -1;
915
916 for (cur_page = 0; cur_page < n_pages; cur_page++) {
917 struct hugepage_file *prev, *cur;
918 int prev_seg_start_page = -1;
919
920 cur = &hugepages[cur_page];
921 prev = cur_page == 0 ? NULL((void*)0) :
922 &hugepages[cur_page - 1];
923
924 new_memseg = 0;
925 end_seg = 0;
926
927 if (cur->size == 0)
928 end_seg = 1;
929 else if (cur->socket_id != (int) socket)
930 end_seg = 1;
931 else if (cur->size != page_sz)
932 end_seg = 1;
933 else if (cur_page == 0)
934 new_memseg = 1;
935#ifdef RTE_ARCH_PPC_64
936 /* On PPC64 architecture, the mmap always start
937 * from higher address to lower address. Here,
938 * physical addresses are in descending order.
939 */
940 else if ((prev->physaddr - cur->physaddr) !=
941 cur->size)
942 new_memseg = 1;
943#else
944 else if ((cur->physaddr - prev->physaddr) !=
945 cur->size)
946 new_memseg = 1;
947#endif
948 if (new_memseg) {
949 /* if we're already inside a segment,
950 * new segment means end of current one
951 */
952 if (seg_start_page != -1) {
953 end_seg = 1;
954 prev_seg_start_page =
955 seg_start_page;
956 }
957 seg_start_page = cur_page;
958 }
959
960 if (end_seg) {
961 if (prev_seg_start_page != -1) {
962 /* we've found a new segment */
963 n_contig_segs++;
964 n_segs += cur_page -
965 prev_seg_start_page;
966 } else if (seg_start_page != -1) {
967 /* we didn't find new segment,
968 * but did end current one
969 */
970 n_contig_segs++;
971 n_segs += cur_page -
972 seg_start_page;
973 seg_start_page = -1;
974 continue;
975 } else {
976 /* we're skipping this page */
977 continue;
978 }
979 }
980 /* segment continues */
981 }
982 /* check if we missed last segment */
983 if (seg_start_page != -1) {
984 n_contig_segs++;
985 n_segs += cur_page - seg_start_page;
986 }
987
988 /* if no segments were found, do not preallocate */
989 if (n_segs == 0)
990 continue;
991
992 /* we now have total number of pages that we will
993 * allocate for this segment list. add separator pages
994 * to the total count, and preallocate VA space.
995 */
996 n_segs += n_contig_segs - 1;
997
998 /* now, preallocate VA space for these segments */
999
1000 /* first, find suitable memseg list for this */
1001 for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS64;
1002 msl_idx++) {
1003 msl = &mcfg->memsegs[msl_idx];
1004
1005 if (msl->base_va != NULL((void*)0))
1006 continue;
1007 break;
1008 }
1009 if (msl_idx == RTE_MAX_MEMSEG_LISTS64) {
1010 RTE_LOG(ERR, EAL, "Not enough space in memseg lists, please increase %s\n",rte_log(4U, 0, "EAL" ": " "Not enough space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS")
1011 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "Not enough space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS");
1012 return -1;
1013 }
1014
1015 /* now, allocate fbarray itself */
1016 if (alloc_memseg_list(msl, page_sz, n_segs, socket,
1017 msl_idx) < 0)
1018 return -1;
1019
1020 /* finally, allocate VA space */
1021 if (alloc_va_space(msl) < 0)
1022 return -1;
1023 }
1024 }
1025 return 0;
1026}
1027
1028/*
1029 * We cannot reallocate memseg lists on the fly because PPC64 stores pages
1030 * backwards, therefore we have to process the entire memseg first before
1031 * remapping it into memseg list VA space.
1032 */
1033static int
1034remap_needed_hugepages(struct hugepage_file *hugepages, int n_pages)
1035{
1036 int cur_page, seg_start_page, new_memseg, ret;
1037
1038 seg_start_page = 0;
1039 for (cur_page = 0; cur_page < n_pages; cur_page++) {
1040 struct hugepage_file *prev, *cur;
1041
1042 new_memseg = 0;
1043
1044 cur = &hugepages[cur_page];
1045 prev = cur_page == 0 ? NULL((void*)0) : &hugepages[cur_page - 1];
1046
1047 /* if size is zero, no more pages left */
1048 if (cur->size == 0)
1049 break;
1050
1051 if (cur_page == 0)
1052 new_memseg = 1;
1053 else if (cur->socket_id != prev->socket_id)
1054 new_memseg = 1;
1055 else if (cur->size != prev->size)
1056 new_memseg = 1;
1057#ifdef RTE_ARCH_PPC_64
1058 /* On PPC64 architecture, the mmap always start from higher
1059 * address to lower address. Here, physical addresses are in
1060 * descending order.
1061 */
1062 else if ((prev->physaddr - cur->physaddr) != cur->size)
1063 new_memseg = 1;
1064#else
1065 else if ((cur->physaddr - prev->physaddr) != cur->size)
1066 new_memseg = 1;
1067#endif
1068
1069 if (new_memseg) {
1070 /* if this isn't the first time, remap segment */
1071 if (cur_page != 0) {
1072 ret = remap_segment(hugepages, seg_start_page,
1073 cur_page);
1074 if (ret != 0)
1075 return -1;
1076 }
1077 /* remember where we started */
1078 seg_start_page = cur_page;
1079 }
1080 /* continuation of previous memseg */
1081 }
1082 /* we were stopped, but we didn't remap the last segment, do it now */
1083 if (cur_page != 0) {
1084 ret = remap_segment(hugepages, seg_start_page,
1085 cur_page);
1086 if (ret != 0)
1087 return -1;
1088 }
1089 return 0;
1090}
1091
1092static inline uint64_t
1093get_socket_mem_size(int socket)
1094{
1095 uint64_t size = 0;
1096 unsigned i;
1097
1098 for (i = 0; i < internal_config.num_hugepage_sizes; i++){
1099 struct hugepage_info *hpi = &internal_config.hugepage_info[i];
1100 size += hpi->hugepage_sz * hpi->num_pages[socket];
1101 }
1102
1103 return size;
1104}
1105
1106/*
1107 * This function is a NUMA-aware equivalent of calc_num_pages.
1108 * It takes in the list of hugepage sizes and the
1109 * number of pages thereof, and calculates the best number of
1110 * pages of each size to fulfill the request for <memory> ram
1111 */
1112static int
1113calc_num_pages_per_socket(uint64_t * memory,
1114 struct hugepage_info *hp_info,
1115 struct hugepage_info *hp_used,
1116 unsigned num_hp_info)
1117{
1118 unsigned socket, j, i = 0;
1119 unsigned requested, available;
1120 int total_num_pages = 0;
1121 uint64_t remaining_mem, cur_mem;
1122 uint64_t total_mem = internal_config.memory;
1123
1124 if (num_hp_info == 0)
1125 return -1;
1126
1127 /* if specific memory amounts per socket weren't requested */
1128 if (internal_config.force_sockets == 0) {
1129 size_t total_size;
1130#ifdef RTE_ARCH_641
1131 int cpu_per_socket[RTE_MAX_NUMA_NODES8];
1132 size_t default_size;
1133 unsigned lcore_id;
1134
1135 /* Compute number of cores per socket */
1136 memset(cpu_per_socket, 0, sizeof(cpu_per_socket));
1137 RTE_LCORE_FOREACH(lcore_id)for (lcore_id = rte_get_next_lcore(-1, 0, 0); lcore_id<128
; lcore_id = rte_get_next_lcore(lcore_id, 0, 0)) {
1138 cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;
1139 }
1140
1141 /*
1142 * Automatically spread requested memory amongst detected sockets according
1143 * to number of cores from cpu mask present on each socket
1144 */
1145 total_size = internal_config.memory;
1146 for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_size != 0; socket++) {
1147
1148 /* Set memory amount per socket */
1149 default_size = (internal_config.memory * cpu_per_socket[socket])
1150 / rte_lcore_count();
1151
1152 /* Limit to maximum available memory on socket */
1153 default_size = RTE_MIN(default_size, get_socket_mem_size(socket))__extension__ ({ __typeof__ (default_size) _a = (default_size
); __typeof__ (get_socket_mem_size(socket)) _b = (get_socket_mem_size
(socket)); _a < _b ? _a : _b; });
1154
1155 /* Update sizes */
1156 memory[socket] = default_size;
1157 total_size -= default_size;
1158 }
1159
1160 /*
1161 * If some memory is remaining, try to allocate it by getting all
1162 * available memory from sockets, one after the other
1163 */
1164 for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_size != 0; socket++) {
1165 /* take whatever is available */
1166 default_size = RTE_MIN(get_socket_mem_size(socket) - memory[socket],__extension__ ({ __typeof__ (get_socket_mem_size(socket) - memory
[socket]) _a = (get_socket_mem_size(socket) - memory[socket])
; __typeof__ (total_size) _b = (total_size); _a < _b ? _a :
_b; })
1167 total_size)__extension__ ({ __typeof__ (get_socket_mem_size(socket) - memory
[socket]) _a = (get_socket_mem_size(socket) - memory[socket])
; __typeof__ (total_size) _b = (total_size); _a < _b ? _a :
_b; });
1168
1169 /* Update sizes */
1170 memory[socket] += default_size;
1171 total_size -= default_size;
1172 }
1173#else
1174 /* in 32-bit mode, allocate all of the memory only on master
1175 * lcore socket
1176 */
1177 total_size = internal_config.memory;
1178 for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_size != 0;
1179 socket++) {
1180 struct rte_config *cfg = rte_eal_get_configuration();
1181 unsigned int master_lcore_socket;
1182
1183 master_lcore_socket =
1184 rte_lcore_to_socket_id(cfg->master_lcore);
1185
1186 if (master_lcore_socket != socket)
1187 continue;
1188
1189 /* Update sizes */
1190 memory[socket] = total_size;
1191 break;
1192 }
1193#endif
1194 }
1195
1196 for (socket = 0; socket < RTE_MAX_NUMA_NODES8 && total_mem != 0; socket++) {
1197 /* skips if the memory on specific socket wasn't requested */
1198 for (i = 0; i < num_hp_info && memory[socket] != 0; i++){
1199 strlcpy(hp_used[i].hugedir, hp_info[i].hugedir,rte_strlcpy(hp_used[i].hugedir, hp_info[i].hugedir, sizeof(hp_used
[i].hugedir))
1200 sizeof(hp_used[i].hugedir))rte_strlcpy(hp_used[i].hugedir, hp_info[i].hugedir, sizeof(hp_used
[i].hugedir));
1201 hp_used[i].num_pages[socket] = RTE_MIN(__extension__ ({ __typeof__ (memory[socket] / hp_info[i].hugepage_sz
) _a = (memory[socket] / hp_info[i].hugepage_sz); __typeof__ (
hp_info[i].num_pages[socket]) _b = (hp_info[i].num_pages[socket
]); _a < _b ? _a : _b; })
1202 memory[socket] / hp_info[i].hugepage_sz,__extension__ ({ __typeof__ (memory[socket] / hp_info[i].hugepage_sz
) _a = (memory[socket] / hp_info[i].hugepage_sz); __typeof__ (
hp_info[i].num_pages[socket]) _b = (hp_info[i].num_pages[socket
]); _a < _b ? _a : _b; })
1203 hp_info[i].num_pages[socket])__extension__ ({ __typeof__ (memory[socket] / hp_info[i].hugepage_sz
) _a = (memory[socket] / hp_info[i].hugepage_sz); __typeof__ (
hp_info[i].num_pages[socket]) _b = (hp_info[i].num_pages[socket
]); _a < _b ? _a : _b; });
1204
1205 cur_mem = hp_used[i].num_pages[socket] *
1206 hp_used[i].hugepage_sz;
1207
1208 memory[socket] -= cur_mem;
1209 total_mem -= cur_mem;
1210
1211 total_num_pages += hp_used[i].num_pages[socket];
1212
1213 /* check if we have met all memory requests */
1214 if (memory[socket] == 0)
1215 break;
1216
1217 /* check if we have any more pages left at this size, if so
1218 * move on to next size */
1219 if (hp_used[i].num_pages[socket] == hp_info[i].num_pages[socket])
1220 continue;
1221 /* At this point we know that there are more pages available that are
1222 * bigger than the memory we want, so lets see if we can get enough
1223 * from other page sizes.
1224 */
1225 remaining_mem = 0;
1226 for (j = i+1; j < num_hp_info; j++)
1227 remaining_mem += hp_info[j].hugepage_sz *
1228 hp_info[j].num_pages[socket];
1229
1230 /* is there enough other memory, if not allocate another page and quit */
1231 if (remaining_mem < memory[socket]){
1232 cur_mem = RTE_MIN(memory[socket],__extension__ ({ __typeof__ (memory[socket]) _a = (memory[socket
]); __typeof__ (hp_info[i].hugepage_sz) _b = (hp_info[i].hugepage_sz
); _a < _b ? _a : _b; })
1233 hp_info[i].hugepage_sz)__extension__ ({ __typeof__ (memory[socket]) _a = (memory[socket
]); __typeof__ (hp_info[i].hugepage_sz) _b = (hp_info[i].hugepage_sz
); _a < _b ? _a : _b; });
1234 memory[socket] -= cur_mem;
1235 total_mem -= cur_mem;
1236 hp_used[i].num_pages[socket]++;
1237 total_num_pages++;
1238 break; /* we are done with this socket*/
1239 }
1240 }
1241 /* if we didn't satisfy all memory requirements per socket */
1242 if (memory[socket] > 0 &&
1243 internal_config.socket_mem[socket] != 0) {
1244 /* to prevent icc errors */
1245 requested = (unsigned) (internal_config.socket_mem[socket] /
1246 0x100000);
1247 available = requested -
1248 ((unsigned) (memory[socket] / 0x100000));
1249 RTE_LOG(ERR, EAL, "Not enough memory available on socket %u! "rte_log(4U, 0, "EAL" ": " "Not enough memory available on socket %u! "
"Requested: %uMB, available: %uMB\n", socket, requested, available
)
1250 "Requested: %uMB, available: %uMB\n", socket,rte_log(4U, 0, "EAL" ": " "Not enough memory available on socket %u! "
"Requested: %uMB, available: %uMB\n", socket, requested, available
)
1251 requested, available)rte_log(4U, 0, "EAL" ": " "Not enough memory available on socket %u! "
"Requested: %uMB, available: %uMB\n", socket, requested, available
);
1252 return -1;
1253 }
1254 }
1255
1256 /* if we didn't satisfy total memory requirements */
1257 if (total_mem > 0) {
1258 requested = (unsigned) (internal_config.memory / 0x100000);
1259 available = requested - (unsigned) (total_mem / 0x100000);
1260 RTE_LOG(ERR, EAL, "Not enough memory available! Requested: %uMB,"rte_log(4U, 0, "EAL" ": " "Not enough memory available! Requested: %uMB,"
" available: %uMB\n", requested, available)
1261 " available: %uMB\n", requested, available)rte_log(4U, 0, "EAL" ": " "Not enough memory available! Requested: %uMB,"
" available: %uMB\n", requested, available);
1262 return -1;
1263 }
1264 return total_num_pages;
1265}
1266
1267static inline size_t
1268eal_get_hugepage_mem_size(void)
1269{
1270 uint64_t size = 0;
1271 unsigned i, j;
1272
1273 for (i = 0; i < internal_config.num_hugepage_sizes; i++) {
1274 struct hugepage_info *hpi = &internal_config.hugepage_info[i];
1275 if (strnlen(hpi->hugedir, sizeof(hpi->hugedir)) != 0) {
1276 for (j = 0; j < RTE_MAX_NUMA_NODES8; j++) {
1277 size += hpi->hugepage_sz * hpi->num_pages[j];
1278 }
1279 }
1280 }
1281
1282 return (size < SIZE_MAX(18446744073709551615UL)) ? (size_t)(size) : SIZE_MAX(18446744073709551615UL);
1283}
1284
1285static struct sigaction huge_action_old;
1286static int huge_need_recover;
1287
1288static void
1289huge_register_sigbus(void)
1290{
1291 sigset_t mask;
1292 struct sigaction action;
1293
1294 sigemptyset(&mask);
1295 sigaddset(&mask, SIGBUS7);
1296 action.sa_flags = 0;
1297 action.sa_mask = mask;
1298 action.sa_handler__sigaction_handler.sa_handler = huge_sigbus_handler;
1299
1300 huge_need_recover = !sigaction(SIGBUS7, &action, &huge_action_old);
1301}
1302
1303static void
1304huge_recover_sigbus(void)
1305{
1306 if (huge_need_recover) {
1307 sigaction(SIGBUS7, &huge_action_old, NULL((void*)0));
1308 huge_need_recover = 0;
1309 }
1310}
1311
1312/*
1313 * Prepare physical memory mapping: fill configuration structure with
1314 * these infos, return 0 on success.
1315 * 1. map N huge pages in separate files in hugetlbfs
1316 * 2. find associated physical addr
1317 * 3. find associated NUMA socket ID
1318 * 4. sort all huge pages by physical address
1319 * 5. remap these N huge pages in the correct order
1320 * 6. unmap the first mapping
1321 * 7. fill memsegs in configuration with contiguous zones
1322 */
1323static int
1324eal_legacy_hugepage_init(void)
1325{
1326 struct rte_mem_config *mcfg;
1327 struct hugepage_file *hugepage = NULL((void*)0), *tmp_hp = NULL((void*)0);
1328 struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES3];
1329 struct rte_fbarray *arr;
1330 struct rte_memseg *ms;
1331
1332 uint64_t memory[RTE_MAX_NUMA_NODES8];
1333
1334 unsigned hp_offset;
1335 int i, j;
1336 int nr_hugefiles, nr_hugepages = 0;
1337 void *addr;
1338
1339 test_phys_addrs_available();
1340
1341 memset(used_hp, 0, sizeof(used_hp));
1342
1343 /* get pointer to global configuration */
1344 mcfg = rte_eal_get_configuration()->mem_config;
1345
1346 /* hugetlbfs can be disabled */
1347 if (internal_config.no_hugetlbfs) {
1348 struct rte_memseg_list *msl;
1349 int n_segs, cur_seg, fd, flags;
1350#ifdef MEMFD_SUPPORTED
1351 int memfd;
1352#endif
1353 uint64_t page_sz;
1354
1355 /* nohuge mode is legacy mode */
1356 internal_config.legacy_mem = 1;
1357
1358 /* nohuge mode is single-file segments mode */
1359 internal_config.single_file_segments = 1;
1360
1361 /* create a memseg list */
1362 msl = &mcfg->memsegs[0];
1363
1364 page_sz = RTE_PGSIZE_4K;
1365 n_segs = internal_config.memory / page_sz;
1366
1367 if (rte_fbarray_init(&msl->memseg_arr, "nohugemem", n_segs,
1368 sizeof(struct rte_memseg))) {
1369 RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list\n");
1370 return -1;
1371 }
1372
1373 /* set up parameters for anonymous mmap */
1374 fd = -1;
1375 flags = MAP_PRIVATE0x02 | MAP_ANONYMOUS0x20;
1376
1377#ifdef MEMFD_SUPPORTED
1378 /* create a memfd and store it in the segment fd table */
1379 memfd = memfd_create("nohuge", 0);
1380 if (memfd < 0) {
1381 RTE_LOG(DEBUG, EAL, "Cannot create memfd: %s\n",rte_log(8U, 0, "EAL" ": " "Cannot create memfd: %s\n", strerror
((*__errno_location ())))
1382 strerror(errno))rte_log(8U, 0, "EAL" ": " "Cannot create memfd: %s\n", strerror
((*__errno_location ())));
1383 RTE_LOG(DEBUG, EAL, "Falling back to anonymous map\n")rte_log(8U, 0, "EAL" ": " "Falling back to anonymous map\n");
1384 } else {
1385 /* we got an fd - now resize it */
1386 if (ftruncate(memfd, internal_config.memory) < 0) {
1387 RTE_LOG(ERR, EAL, "Cannot resize memfd: %s\n",rte_log(4U, 0, "EAL" ": " "Cannot resize memfd: %s\n", strerror
((*__errno_location ())))
1388 strerror(errno))rte_log(4U, 0, "EAL" ": " "Cannot resize memfd: %s\n", strerror
((*__errno_location ())));
1389 RTE_LOG(ERR, EAL, "Falling back to anonymous map\n")rte_log(4U, 0, "EAL" ": " "Falling back to anonymous map\n");
1390 close(memfd);
1391 } else {
1392 /* creating memfd-backed file was successful.
1393 * we want changes to memfd to be visible to
1394 * other processes (such as vhost backend), so
1395 * map it as shared memory.
1396 */
1397 RTE_LOG(DEBUG, EAL, "Using memfd for anonymous memory\n")rte_log(8U, 0, "EAL" ": " "Using memfd for anonymous memory\n"
);
1398 fd = memfd;
1399 flags = MAP_SHARED0x01;
1400 }
1401 }
1402#endif
1403 addr = mmap(NULL((void*)0), internal_config.memory, PROT_READ0x1 | PROT_WRITE0x2,
1404 flags, fd, 0);
1405 if (addr == MAP_FAILED((void *) -1)) {
1406 RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,rte_log(4U, 0, "EAL" ": " "%s: mmap() failed: %s\n", __func__
, strerror((*__errno_location ())))
1407 strerror(errno))rte_log(4U, 0, "EAL" ": " "%s: mmap() failed: %s\n", __func__
, strerror((*__errno_location ())));
1408 return -1;
1409 }
1410 msl->base_va = addr;
1411 msl->page_sz = page_sz;
1412 msl->socket_id = 0;
1413 msl->len = internal_config.memory;
1414
1415 /* we're in single-file segments mode, so only the segment list
1416 * fd needs to be set up.
1417 */
1418 if (fd != -1) {
1419 if (eal_memalloc_set_seg_list_fd(0, fd) < 0) {
1420 RTE_LOG(ERR, EAL, "Cannot set up segment list fd\n")rte_log(4U, 0, "EAL" ": " "Cannot set up segment list fd\n");
1421 /* not a serious error, proceed */
1422 }
1423 }
1424
1425 /* populate memsegs. each memseg is one page long */
1426 for (cur_seg = 0; cur_seg < n_segs; cur_seg++) {
1427 arr = &msl->memseg_arr;
1428
1429 ms = rte_fbarray_get(arr, cur_seg);
1430 if (rte_eal_iova_mode() == RTE_IOVA_VA)
1431 ms->iova = (uintptr_t)addr;
1432 else
1433 ms->iova = RTE_BAD_IOVA((rte_iova_t)-1);
1434 ms->addr = addr;
1435 ms->hugepage_sz = page_sz;
1436 ms->socket_id = 0;
1437 ms->len = page_sz;
1438
1439 rte_fbarray_set_used(arr, cur_seg);
1440
1441 addr = RTE_PTR_ADD(addr, (size_t)page_sz)((void*)((uintptr_t)(addr) + ((size_t)page_sz)));
1442 }
1443 if (mcfg->dma_maskbits &&
1444 rte_mem_check_dma_mask_thread_unsafe(mcfg->dma_maskbits)) {
1445 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n"
, __func__)
1446 "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n",rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n"
, __func__)
1447 __func__)rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n"
, __func__);
1448 if (rte_eal_iova_mode() == RTE_IOVA_VA &&
1449 rte_eal_using_phys_addrs())
1450 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n"
, __func__)
1451 "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n",rte_log(4U, 0, "EAL" ": " "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n"
, __func__)
1452 __func__)rte_log(4U, 0, "EAL" ": " "%s(): Please try initializing EAL with --iova-mode=pa parameter.\n"
, __func__);
1453 goto fail;
1454 }
1455 return 0;
1456 }
1457
1458 /* calculate total number of hugepages available. at this point we haven't
1459 * yet started sorting them so they all are on socket 0 */
1460 for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1461 /* meanwhile, also initialize used_hp hugepage sizes in used_hp */
1462 used_hp[i].hugepage_sz = internal_config.hugepage_info[i].hugepage_sz;
1463
1464 nr_hugepages += internal_config.hugepage_info[i].num_pages[0];
1465 }
1466
1467 /*
1468 * allocate a memory area for hugepage table.
1469 * this isn't shared memory yet. due to the fact that we need some
1470 * processing done on these pages, shared memory will be created
1471 * at a later stage.
1472 */
1473 tmp_hp = malloc(nr_hugepages * sizeof(struct hugepage_file));
1474 if (tmp_hp == NULL((void*)0))
1475 goto fail;
1476
1477 memset(tmp_hp, 0, nr_hugepages * sizeof(struct hugepage_file));
1478
1479 hp_offset = 0; /* where we start the current page size entries */
1480
1481 huge_register_sigbus();
1482
1483 /* make a copy of socket_mem, needed for balanced allocation. */
1484 for (i = 0; i < RTE_MAX_NUMA_NODES8; i++)
1485 memory[i] = internal_config.socket_mem[i];
1486
1487 /* map all hugepages and sort them */
1488 for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){
1489 unsigned pages_old, pages_new;
1490 struct hugepage_info *hpi;
1491
1492 /*
1493 * we don't yet mark hugepages as used at this stage, so
1494 * we just map all hugepages available to the system
1495 * all hugepages are still located on socket 0
1496 */
1497 hpi = &internal_config.hugepage_info[i];
1498
1499 if (hpi->num_pages[0] == 0)
1500 continue;
1501
1502 /* map all hugepages available */
1503 pages_old = hpi->num_pages[0];
1504 pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, memory);
1505 if (pages_new < pages_old) {
1506 RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n"
, pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000
))
1507 "%d not %d hugepages of size %u MB allocated\n",rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n"
, pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000
))
1508 pages_new, pages_old,rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n"
, pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000
))
1509 (unsigned)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "%d not %d hugepages of size %u MB allocated\n"
, pages_new, pages_old, (unsigned)(hpi->hugepage_sz / 0x100000
));
1510
1511 int pages = pages_old - pages_new;
1512
1513 nr_hugepages -= pages;
1514 hpi->num_pages[0] = pages_new;
1515 if (pages_new == 0)
1516 continue;
1517 }
1518
1519 if (phys_addrs_available &&
1520 rte_eal_iova_mode() != RTE_IOVA_VA) {
1521 /* find physical addresses for each hugepage */
1522 if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0) {
1523 RTE_LOG(DEBUG, EAL, "Failed to find phys addr "rte_log(8U, 0, "EAL" ": " "Failed to find phys addr " "for %u MB pages\n"
, (unsigned int)(hpi->hugepage_sz / 0x100000))
1524 "for %u MB pages\n",rte_log(8U, 0, "EAL" ": " "Failed to find phys addr " "for %u MB pages\n"
, (unsigned int)(hpi->hugepage_sz / 0x100000))
1525 (unsigned int)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "Failed to find phys addr " "for %u MB pages\n"
, (unsigned int)(hpi->hugepage_sz / 0x100000));
1526 goto fail;
1527 }
1528 } else {
1529 /* set physical addresses for each hugepage */
1530 if (set_physaddrs(&tmp_hp[hp_offset], hpi) < 0) {
1531 RTE_LOG(DEBUG, EAL, "Failed to set phys addr "rte_log(8U, 0, "EAL" ": " "Failed to set phys addr " "for %u MB pages\n"
, (unsigned int)(hpi->hugepage_sz / 0x100000))
1532 "for %u MB pages\n",rte_log(8U, 0, "EAL" ": " "Failed to set phys addr " "for %u MB pages\n"
, (unsigned int)(hpi->hugepage_sz / 0x100000))
1533 (unsigned int)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "Failed to set phys addr " "for %u MB pages\n"
, (unsigned int)(hpi->hugepage_sz / 0x100000));
1534 goto fail;
1535 }
1536 }
1537
1538 if (find_numasocket(&tmp_hp[hp_offset], hpi) < 0){
1539 RTE_LOG(DEBUG, EAL, "Failed to find NUMA socket for %u MB pages\n",rte_log(8U, 0, "EAL" ": " "Failed to find NUMA socket for %u MB pages\n"
, (unsigned)(hpi->hugepage_sz / 0x100000))
1540 (unsigned)(hpi->hugepage_sz / 0x100000))rte_log(8U, 0, "EAL" ": " "Failed to find NUMA socket for %u MB pages\n"
, (unsigned)(hpi->hugepage_sz / 0x100000));
1541 goto fail;
1542 }
1543
1544 qsort(&tmp_hp[hp_offset], hpi->num_pages[0],
1545 sizeof(struct hugepage_file), cmp_physaddr);
1546
1547 /* we have processed a num of hugepages of this size, so inc offset */
1548 hp_offset += hpi->num_pages[0];
1549 }
1550
1551 huge_recover_sigbus();
1552
1553 if (internal_config.memory == 0 && internal_config.force_sockets == 0)
1554 internal_config.memory = eal_get_hugepage_mem_size();
1555
1556 nr_hugefiles = nr_hugepages;
1557
1558
1559 /* clean out the numbers of pages */
1560 for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++)
1561 for (j = 0; j < RTE_MAX_NUMA_NODES8; j++)
1562 internal_config.hugepage_info[i].num_pages[j] = 0;
1563
1564 /* get hugepages for each socket */
1565 for (i = 0; i < nr_hugefiles; i++) {
1566 int socket = tmp_hp[i].socket_id;
1567
1568 /* find a hugepage info with right size and increment num_pages */
1569 const int nb_hpsizes = RTE_MIN(MAX_HUGEPAGE_SIZES,__extension__ ({ __typeof__ (3) _a = (3); __typeof__ ((int)internal_config
.num_hugepage_sizes) _b = ((int)internal_config.num_hugepage_sizes
); _a < _b ? _a : _b; })
1570 (int)internal_config.num_hugepage_sizes)__extension__ ({ __typeof__ (3) _a = (3); __typeof__ ((int)internal_config
.num_hugepage_sizes) _b = ((int)internal_config.num_hugepage_sizes
); _a < _b ? _a : _b; });
1571 for (j = 0; j < nb_hpsizes; j++) {
1572 if (tmp_hp[i].size ==
1573 internal_config.hugepage_info[j].hugepage_sz) {
1574 internal_config.hugepage_info[j].num_pages[socket]++;
1575 }
1576 }
1577 }
1578
1579 /* make a copy of socket_mem, needed for number of pages calculation */
1580 for (i = 0; i < RTE_MAX_NUMA_NODES8; i++)
1581 memory[i] = internal_config.socket_mem[i];
1582
1583 /* calculate final number of pages */
1584 nr_hugepages = calc_num_pages_per_socket(memory,
1585 internal_config.hugepage_info, used_hp,
1586 internal_config.num_hugepage_sizes);
1587
1588 /* error if not enough memory available */
1589 if (nr_hugepages < 0)
1590 goto fail;
1591
1592 /* reporting in! */
1593 for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1594 for (j = 0; j < RTE_MAX_NUMA_NODES8; j++) {
1595 if (used_hp[i].num_pages[j] > 0) {
1596 RTE_LOG(DEBUG, EAL,rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j)
1597 "Requesting %u pages of size %uMB"rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j)
1598 " from socket %i\n",rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j)
1599 used_hp[i].num_pages[j],rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j)
1600 (unsigned)rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j)
1601 (used_hp[i].hugepage_sz / 0x100000),rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j)
1602 j)rte_log(8U, 0, "EAL" ": " "Requesting %u pages of size %uMB" " from socket %i\n"
, used_hp[i].num_pages[j], (unsigned) (used_hp[i].hugepage_sz
/ 0x100000), j);
1603 }
1604 }
1605 }
1606
1607 /* create shared memory */
1608 hugepage = create_shared_memory(eal_hugepage_data_path(),
1609 nr_hugefiles * sizeof(struct hugepage_file));
1610
1611 if (hugepage == NULL((void*)0)) {
1612 RTE_LOG(ERR, EAL, "Failed to create shared memory!\n")rte_log(4U, 0, "EAL" ": " "Failed to create shared memory!\n"
);
1613 goto fail;
1614 }
1615 memset(hugepage, 0, nr_hugefiles * sizeof(struct hugepage_file));
1616
1617 /*
1618 * unmap pages that we won't need (looks at used_hp).
1619 * also, sets final_va to NULL on pages that were unmapped.
1620 */
1621 if (unmap_unneeded_hugepages(tmp_hp, used_hp,
1622 internal_config.num_hugepage_sizes) < 0) {
1623 RTE_LOG(ERR, EAL, "Unmapping and locking hugepages failed!\n")rte_log(4U, 0, "EAL" ": " "Unmapping and locking hugepages failed!\n"
);
1624 goto fail;
1625 }
1626
1627 /*
1628 * copy stuff from malloc'd hugepage* to the actual shared memory.
1629 * this procedure only copies those hugepages that have orig_va
1630 * not NULL. has overflow protection.
1631 */
1632 if (copy_hugepages_to_shared_mem(hugepage, nr_hugefiles,
1633 tmp_hp, nr_hugefiles) < 0) {
1634 RTE_LOG(ERR, EAL, "Copying tables to shared memory failed!\n")rte_log(4U, 0, "EAL" ": " "Copying tables to shared memory failed!\n"
);
1635 goto fail;
1636 }
1637
1638#ifndef RTE_ARCH_641
1639 /* for legacy 32-bit mode, we did not preallocate VA space, so do it */
1640 if (internal_config.legacy_mem &&
1641 prealloc_segments(hugepage, nr_hugefiles)) {
1642 RTE_LOG(ERR, EAL, "Could not preallocate VA space for hugepages\n")rte_log(4U, 0, "EAL" ": " "Could not preallocate VA space for hugepages\n"
);
1643 goto fail;
1644 }
1645#endif
1646
1647 /* remap all pages we do need into memseg list VA space, so that those
1648 * pages become first-class citizens in DPDK memory subsystem
1649 */
1650 if (remap_needed_hugepages(hugepage, nr_hugefiles)) {
1651 RTE_LOG(ERR, EAL, "Couldn't remap hugepage files into memseg lists\n")rte_log(4U, 0, "EAL" ": " "Couldn't remap hugepage files into memseg lists\n"
);
1652 goto fail;
1653 }
1654
1655 /* free the hugepage backing files */
1656 if (internal_config.hugepage_unlink &&
1657 unlink_hugepage_files(tmp_hp, internal_config.num_hugepage_sizes) < 0) {
1658 RTE_LOG(ERR, EAL, "Unlinking hugepage files failed!\n")rte_log(4U, 0, "EAL" ": " "Unlinking hugepage files failed!\n"
);
1659 goto fail;
1660 }
1661
1662 /* free the temporary hugepage table */
1663 free(tmp_hp);
1664 tmp_hp = NULL((void*)0);
1665
1666 munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file));
1667 hugepage = NULL((void*)0);
1668
1669 /* we're not going to allocate more pages, so release VA space for
1670 * unused memseg lists
1671 */
1672 for (i = 0; i < RTE_MAX_MEMSEG_LISTS64; i++) {
1673 struct rte_memseg_list *msl = &mcfg->memsegs[i];
1674 size_t mem_sz;
1675
1676 /* skip inactive lists */
1677 if (msl->base_va == NULL((void*)0))
1678 continue;
1679 /* skip lists where there is at least one page allocated */
1680 if (msl->memseg_arr.count > 0)
1681 continue;
1682 /* this is an unused list, deallocate it */
1683 mem_sz = msl->len;
1684 munmap(msl->base_va, mem_sz);
1685 msl->base_va = NULL((void*)0);
1686
1687 /* destroy backing fbarray */
1688 rte_fbarray_destroy(&msl->memseg_arr);
1689 }
1690
1691 if (mcfg->dma_maskbits &&
1692 rte_mem_check_dma_mask_thread_unsafe(mcfg->dma_maskbits)) {
1693 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n"
, __func__)
1694 "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n",rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n"
, __func__)
1695 __func__)rte_log(4U, 0, "EAL" ": " "%s(): couldn't allocate memory due to IOVA exceeding limits of current DMA mask.\n"
, __func__);
1696 goto fail;
1697 }
1698
1699 return 0;
1700
1701fail:
1702 huge_recover_sigbus();
1703 free(tmp_hp);
1704 if (hugepage != NULL((void*)0))
1705 munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file));
1706
1707 return -1;
1708}
1709
1710static int __rte_unused__attribute__((__unused__))
1711hugepage_count_walk(const struct rte_memseg_list *msl, void *arg)
1712{
1713 struct hugepage_info *hpi = arg;
1714
1715 if (msl->page_sz != hpi->hugepage_sz)
1716 return 0;
1717
1718 hpi->num_pages[msl->socket_id] += msl->memseg_arr.len;
1719 return 0;
1720}
1721
1722static int
1723limits_callback(int socket_id, size_t cur_limit, size_t new_len)
1724{
1725 RTE_SET_USED(socket_id)(void)(socket_id);
1726 RTE_SET_USED(cur_limit)(void)(cur_limit);
1727 RTE_SET_USED(new_len)(void)(new_len);
1728 return -1;
1729}
1730
1731static int
1732eal_hugepage_init(void)
1733{
1734 struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES3];
1735 uint64_t memory[RTE_MAX_NUMA_NODES8];
1736 int hp_sz_idx, socket_id;
1737
1738 test_phys_addrs_available();
1739
1740 memset(used_hp, 0, sizeof(used_hp));
1741
1742 for (hp_sz_idx = 0;
1743 hp_sz_idx < (int) internal_config.num_hugepage_sizes;
1744 hp_sz_idx++) {
1745#ifndef RTE_ARCH_641
1746 struct hugepage_info dummy;
1747 unsigned int i;
1748#endif
1749 /* also initialize used_hp hugepage sizes in used_hp */
1750 struct hugepage_info *hpi;
1751 hpi = &internal_config.hugepage_info[hp_sz_idx];
1752 used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz;
1753
1754#ifndef RTE_ARCH_641
1755 /* for 32-bit, limit number of pages on socket to whatever we've
1756 * preallocated, as we cannot allocate more.
1757 */
1758 memset(&dummy, 0, sizeof(dummy));
1759 dummy.hugepage_sz = hpi->hugepage_sz;
1760 if (rte_memseg_list_walk(hugepage_count_walk, &dummy) < 0)
1761 return -1;
1762
1763 for (i = 0; i < RTE_DIM(dummy.num_pages)(sizeof (dummy.num_pages) / sizeof ((dummy.num_pages)[0])); i++) {
1764 hpi->num_pages[i] = RTE_MIN(hpi->num_pages[i],__extension__ ({ __typeof__ (hpi->num_pages[i]) _a = (hpi->
num_pages[i]); __typeof__ (dummy.num_pages[i]) _b = (dummy.num_pages
[i]); _a < _b ? _a : _b; })
1765 dummy.num_pages[i])__extension__ ({ __typeof__ (hpi->num_pages[i]) _a = (hpi->
num_pages[i]); __typeof__ (dummy.num_pages[i]) _b = (dummy.num_pages
[i]); _a < _b ? _a : _b; });
1766 }
1767#endif
1768 }
1769
1770 /* make a copy of socket_mem, needed for balanced allocation. */
1771 for (hp_sz_idx = 0; hp_sz_idx < RTE_MAX_NUMA_NODES8; hp_sz_idx++)
1772 memory[hp_sz_idx] = internal_config.socket_mem[hp_sz_idx];
1773
1774 /* calculate final number of pages */
1775 if (calc_num_pages_per_socket(memory,
1776 internal_config.hugepage_info, used_hp,
1777 internal_config.num_hugepage_sizes) < 0)
1778 return -1;
1779
1780 for (hp_sz_idx = 0;
1781 hp_sz_idx < (int)internal_config.num_hugepage_sizes;
1782 hp_sz_idx++) {
1783 for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES8;
1784 socket_id++) {
1785 struct rte_memseg **pages;
1786 struct hugepage_info *hpi = &used_hp[hp_sz_idx];
1787 unsigned int num_pages = hpi->num_pages[socket_id];
1788 unsigned int num_pages_alloc;
1789
1790 if (num_pages == 0)
1791 continue;
1792
1793 RTE_LOG(DEBUG, EAL, "Allocating %u pages of size %" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Allocating %u pages of size %" "l"
"u" "M on socket %i\n", num_pages, hpi->hugepage_sz >>
20, socket_id)
1794 num_pages, hpi->hugepage_sz >> 20, socket_id)rte_log(8U, 0, "EAL" ": " "Allocating %u pages of size %" "l"
"u" "M on socket %i\n", num_pages, hpi->hugepage_sz >>
20, socket_id);
1795
1796 /* we may not be able to allocate all pages in one go,
1797 * because we break up our memory map into multiple
1798 * memseg lists. therefore, try allocating multiple
1799 * times and see if we can get the desired number of
1800 * pages from multiple allocations.
1801 */
1802
1803 num_pages_alloc = 0;
1804 do {
1805 int i, cur_pages, needed;
1806
1807 needed = num_pages - num_pages_alloc;
1808
1809 pages = malloc(sizeof(*pages) * needed);
1810
1811 /* do not request exact number of pages */
1812 cur_pages = eal_memalloc_alloc_seg_bulk(pages,
1813 needed, hpi->hugepage_sz,
1814 socket_id, false0);
1815 if (cur_pages <= 0) {
1816 free(pages);
1817 return -1;
1818 }
1819
1820 /* mark preallocated pages as unfreeable */
1821 for (i = 0; i < cur_pages; i++) {
1822 struct rte_memseg *ms = pages[i];
1823 ms->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE(1 << 0);
1824 }
1825 free(pages);
1826
1827 num_pages_alloc += cur_pages;
1828 } while (num_pages_alloc != num_pages);
1829 }
1830 }
1831 /* if socket limits were specified, set them */
1832 if (internal_config.force_socket_limits) {
1833 unsigned int i;
1834 for (i = 0; i < RTE_MAX_NUMA_NODES8; i++) {
1835 uint64_t limit = internal_config.socket_limit[i];
1836 if (limit == 0)
1837 continue;
1838 if (rte_mem_alloc_validator_register("socket-limit",
1839 limits_callback, i, limit))
1840 RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n")rte_log(4U, 0, "EAL" ": " "Failed to register socket limits validator callback\n"
);
1841 }
1842 }
1843 return 0;
1844}
1845
1846/*
1847 * uses fstat to report the size of a file on disk
1848 */
1849static off_t
1850getFileSize(int fd)
1851{
1852 struct stat st;
1853 if (fstat(fd, &st) < 0)
1854 return 0;
1855 return st.st_size;
1856}
1857
1858/*
1859 * This creates the memory mappings in the secondary process to match that of
1860 * the server process. It goes through each memory segment in the DPDK runtime
1861 * configuration and finds the hugepages which form that segment, mapping them
1862 * in order to form a contiguous block in the virtual memory space
1863 */
1864static int
1865eal_legacy_hugepage_attach(void)
1866{
1867 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
1868 struct hugepage_file *hp = NULL((void*)0);
1869 unsigned int num_hp = 0;
1870 unsigned int i = 0;
1871 unsigned int cur_seg;
1872 off_t size = 0;
1873 int fd, fd_hugepage = -1;
1874
1875 if (aslr_enabled() > 0) {
1876 RTE_LOG(WARNING, EAL, "WARNING: Address Space Layout Randomization "rte_log(5U, 0, "EAL" ": " "WARNING: Address Space Layout Randomization "
"(ASLR) is enabled in the kernel.\n")
1877 "(ASLR) is enabled in the kernel.\n")rte_log(5U, 0, "EAL" ": " "WARNING: Address Space Layout Randomization "
"(ASLR) is enabled in the kernel.\n");
1878 RTE_LOG(WARNING, EAL, " This may cause issues with mapping memory "rte_log(5U, 0, "EAL" ": " " This may cause issues with mapping memory "
"into secondary processes\n")
1879 "into secondary processes\n")rte_log(5U, 0, "EAL" ": " " This may cause issues with mapping memory "
"into secondary processes\n");
1880 }
1881
1882 test_phys_addrs_available();
1883
1884 fd_hugepage = open(eal_hugepage_data_path(), O_RDONLY00);
1885 if (fd_hugepage < 0) {
1886 RTE_LOG(ERR, EAL, "Could not open %s\n",rte_log(4U, 0, "EAL" ": " "Could not open %s\n", eal_hugepage_data_path
())
1887 eal_hugepage_data_path())rte_log(4U, 0, "EAL" ": " "Could not open %s\n", eal_hugepage_data_path
());
1888 goto error;
1889 }
1890
1891 size = getFileSize(fd_hugepage);
1892 hp = mmap(NULL((void*)0), size, PROT_READ0x1, MAP_PRIVATE0x02, fd_hugepage, 0);
1893 if (hp == MAP_FAILED((void *) -1)) {
1894 RTE_LOG(ERR, EAL, "Could not mmap %s\n",rte_log(4U, 0, "EAL" ": " "Could not mmap %s\n", eal_hugepage_data_path
())
1895 eal_hugepage_data_path())rte_log(4U, 0, "EAL" ": " "Could not mmap %s\n", eal_hugepage_data_path
());
1896 goto error;
1897 }
1898
1899 num_hp = size / sizeof(struct hugepage_file);
1900 RTE_LOG(DEBUG, EAL, "Analysing %u files\n", num_hp)rte_log(8U, 0, "EAL" ": " "Analysing %u files\n", num_hp);
1901
1902 /* map all segments into memory to make sure we get the addrs. the
1903 * segments themselves are already in memseg list (which is shared and
1904 * has its VA space already preallocated), so we just need to map
1905 * everything into correct addresses.
1906 */
1907 for (i = 0; i < num_hp; i++) {
1908 struct hugepage_file *hf = &hp[i];
1909 size_t map_sz = hf->size;
1910 void *map_addr = hf->final_va;
1911 int msl_idx, ms_idx;
1912 struct rte_memseg_list *msl;
1913 struct rte_memseg *ms;
1914
1915 /* if size is zero, no more pages left */
1916 if (map_sz == 0)
1917 break;
1918
1919 fd = open(hf->filepath, O_RDWR02);
1920 if (fd < 0) {
1921 RTE_LOG(ERR, EAL, "Could not open %s: %s\n",rte_log(4U, 0, "EAL" ": " "Could not open %s: %s\n", hf->filepath
, strerror((*__errno_location ())))
1922 hf->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Could not open %s: %s\n", hf->filepath
, strerror((*__errno_location ())));
1923 goto error;
1924 }
1925
1926 map_addr = mmap(map_addr, map_sz, PROT_READ0x1 | PROT_WRITE0x2,
1927 MAP_SHARED0x01 | MAP_FIXED0x10, fd, 0);
1928 if (map_addr == MAP_FAILED((void *) -1)) {
1929 RTE_LOG(ERR, EAL, "Could not map %s: %s\n",rte_log(4U, 0, "EAL" ": " "Could not map %s: %s\n", hf->filepath
, strerror((*__errno_location ())))
1930 hf->filepath, strerror(errno))rte_log(4U, 0, "EAL" ": " "Could not map %s: %s\n", hf->filepath
, strerror((*__errno_location ())));
1931 goto fd_error;
1932 }
1933
1934 /* set shared lock on the file. */
1935 if (flock(fd, LOCK_SH1) < 0) {
1936 RTE_LOG(DEBUG, EAL, "%s(): Locking file failed: %s\n",rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed: %s\n", __func__
, strerror((*__errno_location ())))
1937 __func__, strerror(errno))rte_log(8U, 0, "EAL" ": " "%s(): Locking file failed: %s\n", __func__
, strerror((*__errno_location ())));
1938 goto fd_error;
1939 }
1940
1941 /* find segment data */
1942 msl = rte_mem_virt2memseg_list(map_addr);
1943 if (msl == NULL((void*)0)) {
1944 RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg list\n",rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg list\n", __func__
)
1945 __func__)rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg list\n", __func__
);
1946 goto fd_error;
1947 }
1948 ms = rte_mem_virt2memseg(map_addr, msl);
1949 if (ms == NULL((void*)0)) {
1950 RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg\n",rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg\n", __func__
)
1951 __func__)rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg\n", __func__
);
1952 goto fd_error;
1953 }
1954
1955 msl_idx = msl - mcfg->memsegs;
1956 ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
1957 if (ms_idx < 0) {
1958 RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg idx\n",rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg idx\n", __func__
)
1959 __func__)rte_log(8U, 0, "EAL" ": " "%s(): Cannot find memseg idx\n", __func__
);
1960 goto fd_error;
1961 }
1962
1963 /* store segment fd internally */
1964 if (eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd) < 0)
1965 RTE_LOG(ERR, EAL, "Could not store segment fd: %s\n",rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n",
rte_strerror((per_lcore__rte_errno)))
1966 rte_strerror(rte_errno))rte_log(4U, 0, "EAL" ": " "Could not store segment fd: %s\n",
rte_strerror((per_lcore__rte_errno)));
1967 }
1968 /* unmap the hugepage config file, since we are done using it */
1969 munmap(hp, size);
1970 close(fd_hugepage);
1971 return 0;
1972
1973fd_error:
1974 close(fd);
1975error:
1976 /* map all segments into memory to make sure we get the addrs */
1977 cur_seg = 0;
Value stored to 'cur_seg' is never read
1978 for (cur_seg = 0; cur_seg < i; cur_seg++) {
1979 struct hugepage_file *hf = &hp[i];
1980 size_t map_sz = hf->size;
1981 void *map_addr = hf->final_va;
1982
1983 munmap(map_addr, map_sz);
1984 }
1985 if (hp != NULL((void*)0) && hp != MAP_FAILED((void *) -1))
1986 munmap(hp, size);
1987 if (fd_hugepage >= 0)
1988 close(fd_hugepage);
1989 return -1;
1990}
1991
1992static int
1993eal_hugepage_attach(void)
1994{
1995 if (eal_memalloc_sync_with_primary()) {
1996 RTE_LOG(ERR, EAL, "Could not map memory from primary process\n")rte_log(4U, 0, "EAL" ": " "Could not map memory from primary process\n"
);
1997 if (aslr_enabled() > 0)
1998 RTE_LOG(ERR, EAL, "It is recommended to disable ASLR in the kernel and retry running both primary and secondary processes\n")rte_log(4U, 0, "EAL" ": " "It is recommended to disable ASLR in the kernel and retry running both primary and secondary processes\n"
);
1999 return -1;
2000 }
2001 return 0;
2002}
2003
2004int
2005rte_eal_hugepage_init(void)
2006{
2007 return internal_config.legacy_mem ?
2008 eal_legacy_hugepage_init() :
2009 eal_hugepage_init();
2010}
2011
2012int
2013rte_eal_hugepage_attach(void)
2014{
2015 return internal_config.legacy_mem ?
2016 eal_legacy_hugepage_attach() :
2017 eal_hugepage_attach();
2018}
2019
2020int
2021rte_eal_using_phys_addrs(void)
2022{
2023 return phys_addrs_available;
2024}
2025
2026static int __rte_unused__attribute__((__unused__))
2027memseg_primary_init_32(void)
2028{
2029 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
2030 int active_sockets, hpi_idx, msl_idx = 0;
2031 unsigned int socket_id, i;
2032 struct rte_memseg_list *msl;
2033 uint64_t extra_mem_per_socket, total_extra_mem, total_requested_mem;
2034 uint64_t max_mem;
2035
2036 /* no-huge does not need this at all */
2037 if (internal_config.no_hugetlbfs)
2038 return 0;
2039
2040 /* this is a giant hack, but desperate times call for desperate
2041 * measures. in legacy 32-bit mode, we cannot preallocate VA space,
2042 * because having upwards of 2 gigabytes of VA space already mapped will
2043 * interfere with our ability to map and sort hugepages.
2044 *
2045 * therefore, in legacy 32-bit mode, we will be initializing memseg
2046 * lists much later - in eal_memory.c, right after we unmap all the
2047 * unneeded pages. this will not affect secondary processes, as those
2048 * should be able to mmap the space without (too many) problems.
2049 */
2050 if (internal_config.legacy_mem)
2051 return 0;
2052
2053 /* 32-bit mode is a very special case. we cannot know in advance where
2054 * the user will want to allocate their memory, so we have to do some
2055 * heuristics.
2056 */
2057 active_sockets = 0;
2058 total_requested_mem = 0;
2059 if (internal_config.force_sockets)
2060 for (i = 0; i < rte_socket_count(); i++) {
2061 uint64_t mem;
2062
2063 socket_id = rte_socket_id_by_idx(i);
2064 mem = internal_config.socket_mem[socket_id];
2065
2066 if (mem == 0)
2067 continue;
2068
2069 active_sockets++;
2070 total_requested_mem += mem;
2071 }
2072 else
2073 total_requested_mem = internal_config.memory;
2074
2075 max_mem = (uint64_t)RTE_MAX_MEM_MB524288 << 20;
2076 if (total_requested_mem > max_mem) {
2077 RTE_LOG(ERR, EAL, "Invalid parameters: 32-bit process can at most use %uM of memory\n",rte_log(4U, 0, "EAL" ": " "Invalid parameters: 32-bit process can at most use %uM of memory\n"
, (unsigned int)(max_mem >> 20))
2078 (unsigned int)(max_mem >> 20))rte_log(4U, 0, "EAL" ": " "Invalid parameters: 32-bit process can at most use %uM of memory\n"
, (unsigned int)(max_mem >> 20));
2079 return -1;
2080 }
2081 total_extra_mem = max_mem - total_requested_mem;
2082 extra_mem_per_socket = active_sockets == 0 ? total_extra_mem :
2083 total_extra_mem / active_sockets;
2084
2085 /* the allocation logic is a little bit convoluted, but here's how it
2086 * works, in a nutshell:
2087 * - if user hasn't specified on which sockets to allocate memory via
2088 * --socket-mem, we allocate all of our memory on master core socket.
2089 * - if user has specified sockets to allocate memory on, there may be
2090 * some "unused" memory left (e.g. if user has specified --socket-mem
2091 * such that not all memory adds up to 2 gigabytes), so add it to all
2092 * sockets that are in use equally.
2093 *
2094 * page sizes are sorted by size in descending order, so we can safely
2095 * assume that we dispense with bigger page sizes first.
2096 */
2097
2098 /* create memseg lists */
2099 for (i = 0; i < rte_socket_count(); i++) {
2100 int hp_sizes = (int) internal_config.num_hugepage_sizes;
2101 uint64_t max_socket_mem, cur_socket_mem;
2102 unsigned int master_lcore_socket;
2103 struct rte_config *cfg = rte_eal_get_configuration();
2104 bool_Bool skip;
2105
2106 socket_id = rte_socket_id_by_idx(i);
2107
2108#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2109 /* we can still sort pages by socket in legacy mode */
2110 if (!internal_config.legacy_mem && socket_id > 0)
2111 break;
2112#endif
2113
2114 /* if we didn't specifically request memory on this socket */
2115 skip = active_sockets != 0 &&
2116 internal_config.socket_mem[socket_id] == 0;
2117 /* ...or if we didn't specifically request memory on *any*
2118 * socket, and this is not master lcore
2119 */
2120 master_lcore_socket = rte_lcore_to_socket_id(cfg->master_lcore);
2121 skip |= active_sockets == 0 && socket_id != master_lcore_socket;
2122
2123 if (skip) {
2124 RTE_LOG(DEBUG, EAL, "Will not preallocate memory on socket %u\n",rte_log(8U, 0, "EAL" ": " "Will not preallocate memory on socket %u\n"
, socket_id)
2125 socket_id)rte_log(8U, 0, "EAL" ": " "Will not preallocate memory on socket %u\n"
, socket_id);
2126 continue;
2127 }
2128
2129 /* max amount of memory on this socket */
2130 max_socket_mem = (active_sockets != 0 ?
2131 internal_config.socket_mem[socket_id] :
2132 internal_config.memory) +
2133 extra_mem_per_socket;
2134 cur_socket_mem = 0;
2135
2136 for (hpi_idx = 0; hpi_idx < hp_sizes; hpi_idx++) {
2137 uint64_t max_pagesz_mem, cur_pagesz_mem = 0;
2138 uint64_t hugepage_sz;
2139 struct hugepage_info *hpi;
2140 int type_msl_idx, max_segs, total_segs = 0;
2141
2142 hpi = &internal_config.hugepage_info[hpi_idx];
2143 hugepage_sz = hpi->hugepage_sz;
2144
2145 /* check if pages are actually available */
2146 if (hpi->num_pages[socket_id] == 0)
2147 continue;
2148
2149 max_segs = RTE_MAX_MEMSEG_PER_TYPE32768;
2150 max_pagesz_mem = max_socket_mem - cur_socket_mem;
2151
2152 /* make it multiple of page size */
2153 max_pagesz_mem = RTE_ALIGN_FLOOR(max_pagesz_mem,(__typeof__(max_pagesz_mem))((max_pagesz_mem) & (~((__typeof__
(max_pagesz_mem))((hugepage_sz) - 1))))
2154 hugepage_sz)(__typeof__(max_pagesz_mem))((max_pagesz_mem) & (~((__typeof__
(max_pagesz_mem))((hugepage_sz) - 1))));
2155
2156 RTE_LOG(DEBUG, EAL, "Attempting to preallocate "rte_log(8U, 0, "EAL" ": " "Attempting to preallocate " "%" "l"
"u" "M on socket %i\n", max_pagesz_mem >> 20, socket_id
)
2157 "%" PRIu64 "M on socket %i\n",rte_log(8U, 0, "EAL" ": " "Attempting to preallocate " "%" "l"
"u" "M on socket %i\n", max_pagesz_mem >> 20, socket_id
)
2158 max_pagesz_mem >> 20, socket_id)rte_log(8U, 0, "EAL" ": " "Attempting to preallocate " "%" "l"
"u" "M on socket %i\n", max_pagesz_mem >> 20, socket_id
);
2159
2160 type_msl_idx = 0;
2161 while (cur_pagesz_mem < max_pagesz_mem &&
2162 total_segs < max_segs) {
2163 uint64_t cur_mem;
2164 unsigned int n_segs;
2165
2166 if (msl_idx >= RTE_MAX_MEMSEG_LISTS64) {
2167 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS")
2168 "No more space in memseg lists, please increase %s\n",rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS")
2169 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS");
2170 return -1;
2171 }
2172
2173 msl = &mcfg->memsegs[msl_idx];
2174
2175 cur_mem = get_mem_amount(hugepage_sz,
2176 max_pagesz_mem);
2177 n_segs = cur_mem / hugepage_sz;
2178
2179 if (alloc_memseg_list(msl, hugepage_sz, n_segs,
2180 socket_id, type_msl_idx)) {
2181 /* failing to allocate a memseg list is
2182 * a serious error.
2183 */
2184 RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate memseg list\n");
2185 return -1;
2186 }
2187
2188 if (alloc_va_space(msl)) {
2189 /* if we couldn't allocate VA space, we
2190 * can try with smaller page sizes.
2191 */
2192 RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list, retrying with different page size\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space for memseg list, retrying with different page size\n"
);
2193 /* deallocate memseg list */
2194 if (free_memseg_list(msl))
2195 return -1;
2196 break;
2197 }
2198
2199 total_segs += msl->memseg_arr.len;
2200 cur_pagesz_mem = total_segs * hugepage_sz;
2201 type_msl_idx++;
2202 msl_idx++;
2203 }
2204 cur_socket_mem += cur_pagesz_mem;
2205 }
2206 if (cur_socket_mem == 0) {
2207 RTE_LOG(ERR, EAL, "Cannot allocate VA space on socket %u\n",rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space on socket %u\n"
, socket_id)
2208 socket_id)rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space on socket %u\n"
, socket_id);
2209 return -1;
2210 }
2211 }
2212
2213 return 0;
2214}
2215
2216static int __rte_unused__attribute__((__unused__))
2217memseg_primary_init(void)
2218{
2219 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
2220 struct memtype {
2221 uint64_t page_sz;
2222 int socket_id;
2223 } *memtypes = NULL((void*)0);
2224 int i, hpi_idx, msl_idx, ret = -1; /* fail unless told to succeed */
2225 struct rte_memseg_list *msl;
2226 uint64_t max_mem, max_mem_per_type;
2227 unsigned int max_seglists_per_type;
2228 unsigned int n_memtypes, cur_type;
2229
2230 /* no-huge does not need this at all */
2231 if (internal_config.no_hugetlbfs)
2232 return 0;
2233
2234 /*
2235 * figuring out amount of memory we're going to have is a long and very
2236 * involved process. the basic element we're operating with is a memory
2237 * type, defined as a combination of NUMA node ID and page size (so that
2238 * e.g. 2 sockets with 2 page sizes yield 4 memory types in total).
2239 *
2240 * deciding amount of memory going towards each memory type is a
2241 * balancing act between maximum segments per type, maximum memory per
2242 * type, and number of detected NUMA nodes. the goal is to make sure
2243 * each memory type gets at least one memseg list.
2244 *
2245 * the total amount of memory is limited by RTE_MAX_MEM_MB value.
2246 *
2247 * the total amount of memory per type is limited by either
2248 * RTE_MAX_MEM_MB_PER_TYPE, or by RTE_MAX_MEM_MB divided by the number
2249 * of detected NUMA nodes. additionally, maximum number of segments per
2250 * type is also limited by RTE_MAX_MEMSEG_PER_TYPE. this is because for
2251 * smaller page sizes, it can take hundreds of thousands of segments to
2252 * reach the above specified per-type memory limits.
2253 *
2254 * additionally, each type may have multiple memseg lists associated
2255 * with it, each limited by either RTE_MAX_MEM_MB_PER_LIST for bigger
2256 * page sizes, or RTE_MAX_MEMSEG_PER_LIST segments for smaller ones.
2257 *
2258 * the number of memseg lists per type is decided based on the above
2259 * limits, and also taking number of detected NUMA nodes, to make sure
2260 * that we don't run out of memseg lists before we populate all NUMA
2261 * nodes with memory.
2262 *
2263 * we do this in three stages. first, we collect the number of types.
2264 * then, we figure out memory constraints and populate the list of
2265 * would-be memseg lists. then, we go ahead and allocate the memseg
2266 * lists.
2267 */
2268
2269 /* create space for mem types */
2270 n_memtypes = internal_config.num_hugepage_sizes * rte_socket_count();
2271 memtypes = calloc(n_memtypes, sizeof(*memtypes));
2272 if (memtypes == NULL((void*)0)) {
2273 RTE_LOG(ERR, EAL, "Cannot allocate space for memory types\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate space for memory types\n"
);
2274 return -1;
2275 }
2276
2277 /* populate mem types */
2278 cur_type = 0;
2279 for (hpi_idx = 0; hpi_idx < (int) internal_config.num_hugepage_sizes;
2280 hpi_idx++) {
2281 struct hugepage_info *hpi;
2282 uint64_t hugepage_sz;
2283
2284 hpi = &internal_config.hugepage_info[hpi_idx];
2285 hugepage_sz = hpi->hugepage_sz;
2286
2287 for (i = 0; i < (int) rte_socket_count(); i++, cur_type++) {
2288 int socket_id = rte_socket_id_by_idx(i);
2289
2290#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2291 /* we can still sort pages by socket in legacy mode */
2292 if (!internal_config.legacy_mem && socket_id > 0)
2293 break;
2294#endif
2295 memtypes[cur_type].page_sz = hugepage_sz;
2296 memtypes[cur_type].socket_id = socket_id;
2297
2298 RTE_LOG(DEBUG, EAL, "Detected memory type: "rte_log(8U, 0, "EAL" ": " "Detected memory type: " "socket_id:%u hugepage_sz:%"
"l" "u" "\n", socket_id, hugepage_sz)
2299 "socket_id:%u hugepage_sz:%" PRIu64 "\n",rte_log(8U, 0, "EAL" ": " "Detected memory type: " "socket_id:%u hugepage_sz:%"
"l" "u" "\n", socket_id, hugepage_sz)
2300 socket_id, hugepage_sz)rte_log(8U, 0, "EAL" ": " "Detected memory type: " "socket_id:%u hugepage_sz:%"
"l" "u" "\n", socket_id, hugepage_sz);
2301 }
2302 }
2303 /* number of memtypes could have been lower due to no NUMA support */
2304 n_memtypes = cur_type;
2305
2306 /* set up limits for types */
2307 max_mem = (uint64_t)RTE_MAX_MEM_MB524288 << 20;
2308 max_mem_per_type = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20,__extension__ ({ __typeof__ ((uint64_t)131072 << 20) _a
= ((uint64_t)131072 << 20); __typeof__ (max_mem / n_memtypes
) _b = (max_mem / n_memtypes); _a < _b ? _a : _b; })
2309 max_mem / n_memtypes)__extension__ ({ __typeof__ ((uint64_t)131072 << 20) _a
= ((uint64_t)131072 << 20); __typeof__ (max_mem / n_memtypes
) _b = (max_mem / n_memtypes); _a < _b ? _a : _b; });
2310 /*
2311 * limit maximum number of segment lists per type to ensure there's
2312 * space for memseg lists for all NUMA nodes with all page sizes
2313 */
2314 max_seglists_per_type = RTE_MAX_MEMSEG_LISTS64 / n_memtypes;
2315
2316 if (max_seglists_per_type == 0) {
2317 RTE_LOG(ERR, EAL, "Cannot accommodate all memory types, please increase %s\n",rte_log(4U, 0, "EAL" ": " "Cannot accommodate all memory types, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS")
2318 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "Cannot accommodate all memory types, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS");
2319 goto out;
2320 }
2321
2322 /* go through all mem types and create segment lists */
2323 msl_idx = 0;
2324 for (cur_type = 0; cur_type < n_memtypes; cur_type++) {
2325 unsigned int cur_seglist, n_seglists, n_segs;
2326 unsigned int max_segs_per_type, max_segs_per_list;
2327 struct memtype *type = &memtypes[cur_type];
2328 uint64_t max_mem_per_list, pagesz;
2329 int socket_id;
2330
2331 pagesz = type->page_sz;
2332 socket_id = type->socket_id;
2333
2334 /*
2335 * we need to create segment lists for this type. we must take
2336 * into account the following things:
2337 *
2338 * 1. total amount of memory we can use for this memory type
2339 * 2. total amount of memory per memseg list allowed
2340 * 3. number of segments needed to fit the amount of memory
2341 * 4. number of segments allowed per type
2342 * 5. number of segments allowed per memseg list
2343 * 6. number of memseg lists we are allowed to take up
2344 */
2345
2346 /* calculate how much segments we will need in total */
2347 max_segs_per_type = max_mem_per_type / pagesz;
2348 /* limit number of segments to maximum allowed per type */
2349 max_segs_per_type = RTE_MIN(max_segs_per_type,__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type
); __typeof__ ((unsigned int)32768) _b = ((unsigned int)32768
); _a < _b ? _a : _b; })
2350 (unsigned int)RTE_MAX_MEMSEG_PER_TYPE)__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type
); __typeof__ ((unsigned int)32768) _b = ((unsigned int)32768
); _a < _b ? _a : _b; });
2351 /* limit number of segments to maximum allowed per list */
2352 max_segs_per_list = RTE_MIN(max_segs_per_type,__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type
); __typeof__ ((unsigned int)8192) _b = ((unsigned int)8192);
_a < _b ? _a : _b; })
2353 (unsigned int)RTE_MAX_MEMSEG_PER_LIST)__extension__ ({ __typeof__ (max_segs_per_type) _a = (max_segs_per_type
); __typeof__ ((unsigned int)8192) _b = ((unsigned int)8192);
_a < _b ? _a : _b; });
2354
2355 /* calculate how much memory we can have per segment list */
2356 max_mem_per_list = RTE_MIN(max_segs_per_list * pagesz,__extension__ ({ __typeof__ (max_segs_per_list * pagesz) _a =
(max_segs_per_list * pagesz); __typeof__ ((uint64_t)32768 <<
20) _b = ((uint64_t)32768 << 20); _a < _b ? _a : _b
; })
2357 (uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20)__extension__ ({ __typeof__ (max_segs_per_list * pagesz) _a =
(max_segs_per_list * pagesz); __typeof__ ((uint64_t)32768 <<
20) _b = ((uint64_t)32768 << 20); _a < _b ? _a : _b
; });
2358
2359 /* calculate how many segments each segment list will have */
2360 n_segs = RTE_MIN(max_segs_per_list, max_mem_per_list / pagesz)__extension__ ({ __typeof__ (max_segs_per_list) _a = (max_segs_per_list
); __typeof__ (max_mem_per_list / pagesz) _b = (max_mem_per_list
/ pagesz); _a < _b ? _a : _b; });
2361
2362 /* calculate how many segment lists we can have */
2363 n_seglists = RTE_MIN(max_segs_per_type / n_segs,__extension__ ({ __typeof__ (max_segs_per_type / n_segs) _a =
(max_segs_per_type / n_segs); __typeof__ (max_mem_per_type /
max_mem_per_list) _b = (max_mem_per_type / max_mem_per_list)
; _a < _b ? _a : _b; })
2364 max_mem_per_type / max_mem_per_list)__extension__ ({ __typeof__ (max_segs_per_type / n_segs) _a =
(max_segs_per_type / n_segs); __typeof__ (max_mem_per_type /
max_mem_per_list) _b = (max_mem_per_type / max_mem_per_list)
; _a < _b ? _a : _b; });
2365
2366 /* limit number of segment lists according to our maximum */
2367 n_seglists = RTE_MIN(n_seglists, max_seglists_per_type)__extension__ ({ __typeof__ (n_seglists) _a = (n_seglists); __typeof__
(max_seglists_per_type) _b = (max_seglists_per_type); _a <
_b ? _a : _b; });
2368
2369 RTE_LOG(DEBUG, EAL, "Creating %i segment lists: "rte_log(8U, 0, "EAL" ": " "Creating %i segment lists: " "n_segs:%i socket_id:%i hugepage_sz:%"
"l" "u" "\n", n_seglists, n_segs, socket_id, pagesz)
2370 "n_segs:%i socket_id:%i hugepage_sz:%" PRIu64 "\n",rte_log(8U, 0, "EAL" ": " "Creating %i segment lists: " "n_segs:%i socket_id:%i hugepage_sz:%"
"l" "u" "\n", n_seglists, n_segs, socket_id, pagesz)
2371 n_seglists, n_segs, socket_id, pagesz)rte_log(8U, 0, "EAL" ": " "Creating %i segment lists: " "n_segs:%i socket_id:%i hugepage_sz:%"
"l" "u" "\n", n_seglists, n_segs, socket_id, pagesz);
2372
2373 /* create all segment lists */
2374 for (cur_seglist = 0; cur_seglist < n_seglists; cur_seglist++) {
2375 if (msl_idx >= RTE_MAX_MEMSEG_LISTS64) {
2376 RTE_LOG(ERR, EAL,rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS")
2377 "No more space in memseg lists, please increase %s\n",rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS")
2378 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS))rte_log(4U, 0, "EAL" ": " "No more space in memseg lists, please increase %s\n"
, "CONFIG_RTE_MAX_MEMSEG_LISTS");
2379 goto out;
2380 }
2381 msl = &mcfg->memsegs[msl_idx++];
2382
2383 if (alloc_memseg_list(msl, pagesz, n_segs,
2384 socket_id, cur_seglist))
2385 goto out;
2386
2387 if (alloc_va_space(msl)) {
2388 RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n")rte_log(4U, 0, "EAL" ": " "Cannot allocate VA space for memseg list\n"
);
2389 goto out;
2390 }
2391 }
2392 }
2393 /* we're successful */
2394 ret = 0;
2395out:
2396 free(memtypes);
2397 return ret;
2398}
2399
2400static int
2401memseg_secondary_init(void)
2402{
2403 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
2404 int msl_idx = 0;
2405 struct rte_memseg_list *msl;
2406
2407 for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS64; msl_idx++) {
2408
2409 msl = &mcfg->memsegs[msl_idx];
2410
2411 /* skip empty memseg lists */
2412 if (msl->memseg_arr.len == 0)
2413 continue;
2414
2415 if (rte_fbarray_attach(&msl->memseg_arr)) {
2416 RTE_LOG(ERR, EAL, "Cannot attach to primary process memseg lists\n")rte_log(4U, 0, "EAL" ": " "Cannot attach to primary process memseg lists\n"
);
2417 return -1;
2418 }
2419
2420 /* preallocate VA space */
2421 if (alloc_va_space(msl)) {
2422 RTE_LOG(ERR, EAL, "Cannot preallocate VA space for hugepage memory\n")rte_log(4U, 0, "EAL" ": " "Cannot preallocate VA space for hugepage memory\n"
);
2423 return -1;
2424 }
2425 }
2426
2427 return 0;
2428}
2429
2430int
2431rte_eal_memseg_init(void)
2432{
2433 /* increase rlimit to maximum */
2434 struct rlimit lim;
2435
2436 if (getrlimit(RLIMIT_NOFILERLIMIT_NOFILE, &lim) == 0) {
2437 /* set limit to maximum */
2438 lim.rlim_cur = lim.rlim_max;
2439
2440 if (setrlimit(RLIMIT_NOFILERLIMIT_NOFILE, &lim) < 0) {
2441 RTE_LOG(DEBUG, EAL, "Setting maximum number of open files failed: %s\n",rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files failed: %s\n"
, strerror((*__errno_location ())))
2442 strerror(errno))rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files failed: %s\n"
, strerror((*__errno_location ())));
2443 } else {
2444 RTE_LOG(DEBUG, EAL, "Setting maximum number of open files to %"rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files to %"
"l" "u" "\n", (uint64_t)lim.rlim_cur)
2445 PRIu64 "\n",rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files to %"
"l" "u" "\n", (uint64_t)lim.rlim_cur)
2446 (uint64_t)lim.rlim_cur)rte_log(8U, 0, "EAL" ": " "Setting maximum number of open files to %"
"l" "u" "\n", (uint64_t)lim.rlim_cur);
2447 }
2448 } else {
2449 RTE_LOG(ERR, EAL, "Cannot get current resource limits\n")rte_log(4U, 0, "EAL" ": " "Cannot get current resource limits\n"
);
2450 }
2451#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2452 if (!internal_config.legacy_mem && rte_socket_count() > 1) {
2453 RTE_LOG(WARNING, EAL, "DPDK is running on a NUMA system, but is compiled without NUMA support.\n")rte_log(5U, 0, "EAL" ": " "DPDK is running on a NUMA system, but is compiled without NUMA support.\n"
);
2454 RTE_LOG(WARNING, EAL, "This will have adverse consequences for performance and usability.\n")rte_log(5U, 0, "EAL" ": " "This will have adverse consequences for performance and usability.\n"
);
2455 RTE_LOG(WARNING, EAL, "Please use --"OPT_LEGACY_MEM" option, or recompile with NUMA support.\n")rte_log(5U, 0, "EAL" ": " "Please use --""legacy-mem"" option, or recompile with NUMA support.\n"
);
2456 }
2457#endif
2458
2459 return rte_eal_process_type() == RTE_PROC_PRIMARY ?
2460#ifndef RTE_ARCH_641
2461 memseg_primary_init_32() :
2462#else
2463 memseg_primary_init() :
2464#endif
2465 memseg_secondary_init();
2466}